From 9db147715b036c6c9f645d3bb2dc5c21603ab635 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 2 Mar 2018 17:43:12 +0100 Subject: [PATCH] Fix Ceph RBD image - create 120MB volume instead of 1MB for volume_io tests - rebase to Fedora 26 - added compatibility with ext4 and older ceph clients - unify CephFS and Ceph RBD images. --- test/e2e/storage/volume_io.go | 2 +- test/e2e/storage/volumes.go | 5 ++-- test/images/volumes-tester/rbd/Dockerfile | 5 ++-- test/images/volumes-tester/rbd/Makefile | 2 +- test/images/volumes-tester/rbd/block.tar.gz | Bin 1675 -> 128753 bytes test/images/volumes-tester/rbd/bootstrap.sh | 26 ++++++++++++++++-- test/images/volumes-tester/rbd/ceph.conf.sh | 6 +++- .../images/volumes-tester/rbd/create_block.sh | 5 ++-- test/images/volumes-tester/rbd/mds.sh | 20 ++++++++++++++ test/utils/image/manifest.go | 2 +- 10 files changed, 59 insertions(+), 14 deletions(-) create mode 100755 test/images/volumes-tester/rbd/mds.sh diff --git a/test/e2e/storage/volume_io.go b/test/e2e/storage/volume_io.go index e9205e4dfc..37b5c5d2d4 100644 --- a/test/e2e/storage/volume_io.go +++ b/test/e2e/storage/volume_io.go @@ -416,7 +416,7 @@ var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() { Name: name, }, FSType: "ext2", - ReadOnly: true, + ReadOnly: false, }, } }) diff --git a/test/e2e/storage/volumes.go b/test/e2e/storage/volumes.go index ec76c1ac3b..2ed8023d96 100644 --- a/test/e2e/storage/volumes.go +++ b/test/e2e/storage/volumes.go @@ -279,10 +279,9 @@ var _ = utils.SIGDescribe("Volumes", func() { ObjectMeta: metav1.ObjectMeta{ Name: config.Prefix + "-secret", }, - // Must use the ceph keyring at contrib/for-tests/volumes-ceph/ceph/init.sh - // and encode in base64 + // from test/images/volumes-tester/rbd/keyring Data: map[string][]byte{ - "key": []byte("AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ=="), + "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), }, Type: "kubernetes.io/cephfs", } diff --git a/test/images/volumes-tester/rbd/Dockerfile b/test/images/volumes-tester/rbd/Dockerfile index e195f3f064..2c34651915 100644 --- a/test/images/volumes-tester/rbd/Dockerfile +++ b/test/images/volumes-tester/rbd/Dockerfile @@ -15,16 +15,17 @@ # CEPH all in one # Based on image by Ricardo Rocha, ricardo@catalyst.net.nz -FROM fedora +FROM fedora:26 # Base Packages -RUN yum install -y wget ceph ceph-fuse strace && yum clean all +RUN yum install -y wget strace psmisc procps-ng ceph ceph-fuse && yum clean all # Get ports exposed EXPOSE 6789 ADD ./bootstrap.sh /bootstrap.sh ADD ./mon.sh /mon.sh +ADD ./mds.sh /mds.sh ADD ./osd.sh /osd.sh ADD ./ceph.conf.sh /ceph.conf.sh ADD ./keyring /var/lib/ceph/mon/keyring diff --git a/test/images/volumes-tester/rbd/Makefile b/test/images/volumes-tester/rbd/Makefile index fc35634606..910ba81605 100644 --- a/test/images/volumes-tester/rbd/Makefile +++ b/test/images/volumes-tester/rbd/Makefile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -TAG = 0.1 +TAG = 0.2 PREFIX = staging-k8s.gcr.io all: push diff --git a/test/images/volumes-tester/rbd/block.tar.gz b/test/images/volumes-tester/rbd/block.tar.gz index 395d1492f9bbee5843978fc6cb66def3ea585a70..8ec08a1d96204c7a7b1a8b517b3eb8bbcb2dc003 100644 GIT binary patch literal 128753 zcmeI*d0bOh+6V9`)@W_TE)I2ELR%FrqqY?cYoan$!I89BSC&YvQWgmcvV=fT+p(i# zr9?zUh^=7Cpio6w0wkzI5oOVUA_fR`gUBWj2qYwVY2V@8_cQ;z{Y=}b)nESO-1|J| zdA|4DbNPIX7smc^GB`oBa%LBo_)*+ui9}2VdYa9Gy+oJmP%O5?6 zk$3gyXQ|qBL$bnHxiYp|l_ku2MceMEW+ijZG;XiBv79>)>eb$|`5;TTna7B#{j#j- z@i&`!krijlg2NdF`Xg`h&ifa#OBr>aRXK#3#`~Smwco6g8N}@dV@?arq-tOnZwU;o zy~#|rDh>$j?aW~&pV`mmF*-h=^A=SLtCd|Fw{4rB-~2__9qD7kBU!8W0)FFUg}wSa zf5(nAx_9@Ph(c8|uTI>1JE=&gp?lriX6nA2f6-XemM&K)|<^y+(1Eh|k-L?5ycw#!)_EVQ!7P^q+MMSz@)&T5P&Cie)Nd zbu_dmaOF39ojNpTPHd~{nksvGB&)}%iE*SnXYWTh7T^3a ztb?(3``}^zAN_A_Gkv?yg+ymvCx<^F(UUz#lCuiaQqe#g-%5s4b{CMr zb-^QL3>1-hGSAo6tjG!0?_J|{fg}nbF=7>&ypO~z_=F_7L>PFR#MQ^3nOz+BUfU35Xp0Z^cbudsbV{|G}e)5 z!A=rWN0oM-%os=|xXm(MgiXkqmZHMp}|6 znk1S=*7^++;!c*j9aUNy2^LW&lE&<)`zV!QXvr2}cZO8fL>(s4x2g5QCw(=M31g^` zZR9#}$%#|QVZYpw71&NLs9h$Rn%bOX)c(;!y@yU2{@q{YY( zQjk4$n6wy4ZDU%p(w$C|lzM_XqiN*IcYjW5V~|S{`7xQtoU}XiJ~c?X7+OIF4^n56 zji~e_X)Tu8iEfbE+0LjxJ4I5D-y9;F7!r5Y$8gJq-+r!3NXH_6T+T@FwW z1yaU#YMXn7Xr+?-r29UZhD+rsC4&;O=7&58%xaP-wty70j)I|)h^HhnA!?Qul>98T28$c{lZNN}>``YilwoJ)1m9BG-`8GpUqhKj^+gO0Od+ zhp1<58;NP6`Y9vx@ThrKlX=9{Lhz>ssq9f?kV~Dpg&HJhR*>3yNf|~dajT@tpzfMv z5|c;ng(7Oj@JOO2Y8&{1#K_4re}HcC9l{_v|~?m5$$cA>Ri zoplwN<~lYqfA$WHx)wK5xZ>E73c=vGKKwOxu(1*LbK?^O+S8Se`W zHyIb_tJ`>W55LY+WSI?3FiyzQ_lOR%uhhLkd!{}n^B{=}(^I*8NUu~Xm*3A{+9bRB zFuucCU)3Nqtk$Mn(B9|g))&RL#Gk3@PZeCPU)~UCQb*6PF`u4)D4@kcWh#tdNrjtU zDxBYDPRp;j%|3n2=Z&j2a9F1 zVgjWu(#^WuFiBWTSdykgouzhEM4T#2FO)>5-CNDEN_(ZoRee9q&B3;DVWV}*;m(v! z(WC5*DXIOZV$?EqtorX^TknXg+*FA&$CQ*#@r-zxRSb{6iT^%7#KFpWrn5~9m%m`#&zSYk{^$qWeDJJxwn)n&X-VO_63O%!j{(@5}#yzg=h0(ADNOD-|nit|_k7SSze+G8LIMfr`MIHU%TS zPQpxMyYv3Sa}y>QZ44W9y&4Pkf*Nl6(JGEGrRjK6f^l}SZSg$AHhwgJxvo&7Ri~&K zib4hZl+ev|+&HV)vDnHG$mj97{QrAbT5db1KXZn^SwG#A;mqU}b-4sSVze}0E6$KP zd*xU1V;|0qc2`EQ2>Gf1Rwwb z2tWV=5P$##AOHafKmY;|fB*y_009U<00K{}fbl?zZry=5_MN0Te zH+9v*6dygSfq9xPZ#kU6%G_V@MM6RYyZ^z%4$iu=?9?JdNL!OSTd20_5MO03V5JJQ zJy*M1tWu{(g$#E0r6eZ_cjmMV_?eFhUE4$VULn0-c69IllUscwS{P;^de5;(ex4{cBHsRO((!6bB=JMZ5wJQ4bIFo&4*UYCG3%xz10;VI%^0HIM9Y45wFK<=7 zY>nGOrMRPa(^+!}Rawjsu_v#X`SLy@?#s>R7IV2AdKq>ah;njw$W=o$|= zTCq)EHQLDE;yOnQhbntlE*H;;-qG4rx_4bfu#fRZW#_$Y=SdYCDkn(e&zI+|)>F3oGg=EZt{r1_eHvG9M(gm44jiVm zGHa}CH~b?y`YFA5Ke4SNw$AYrdSeA~s*rElGln=14Ys3yVkl0700bZa0SG_<0uX=z z1Rwwb2tWV=5P$##AOHafKmY;|fB*!3p8_Qj;g3#?rO`g!b!5y?=5zF-6_#JE2@${e z`|Ji)KXU^4vpmnH(BJ+>=rr(gG~VU^%w2yp3;_s000Izz00bZa0SG_<0uX=z1Rwwb z2tWV=5P$##AOHaf{B(g=9s8;`+P=DX)WH4JzkYpY+-}Ryg}NzT4W~QZC%$3V5KE&o zcgDXUEwn2*zvpvq$$J+a?Z)kP@0_>q?)}f@JpG!0qO6Mp*C zI1>UtPhezvq@le% z2ZO^I!j4^=ssenjf0njNmLiqgy)~gwpQv$7p2jNfwYc5L%pCkqbnH*!dS%^((=$8n zln9ar)56Vqrw+f{;J~gy*N4OJEI6XN>ZS;Ov7+U*S+?~NvFYW;OE`~oi`lN?+VaKnqpXZ_k4ELQ{pU?ePOEo$9ns!18~W=WLwB?A z-LCAHe%`7(?A8Z8GyS8{-$bBfUj4ofr?@4Cdl#*gfn$D?8ZapYAOHaf zKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwx`XG-A2?F`|~IbYj7UlP&3b-m3? zOD2uJD|Xs*Amv8W)um(hSDLNAo$*=FSH%YsS9yFsFnZDR!M?AL+gNWk=OydRuuJ?) zbAnbZi$7uA6LZ=3Nc2N`sPEQiiVw~HCj!QIA`&>O?B!lpMG>p&3%uPHD#dL*A-BC{ zcLsM9A5v=`)pg1`rM;oS?@USZEr{>6%qTD3tYikq6urwI9H`VMM{r_{E`|QFYh&VwO-biG`bNSSe#b$+gv$}v_NB%qurfrcu?u57M5R(W{Xe03 zYGUFx)VLJbb(PTvUfip{chU556Zg%hra3eM0SG_<0uX=z1Rwwb2tWV=5P$##AOHaf zKmY;|fB*y_009X6P6Pygl8FH{+TzzfcP)``3@O<^=H-pgY3qL{)uYV+u0WjUxyi~X z+8$lm9!_rExc~0tm<<9DfB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5P$## zeulu}v#U*~*SKtdEsoRaKj*^aC6mT z{SGGew60p6%R8*7==ak#aN8GfgMLbr z$hV0#KEA1J%i{?C^`--XuJd-ApT$8ZdtIcu^E3Lo@L-?88-F5=dgsip!D%PcX6vi67u5?z2Bw+5THsOFFXnxx7CFplE!4PnjMW{`xHi7l zYBfAErBx;21f28I_CLv&;McG1?LfX|ON(KECjWku&}l$ip~oM*`1RE>0|X!d0SG_< z0uX=z1Rwwb2tWV=5P$##AOHafKmY;|fB*y_@aqV~b+y+nd67mNXZFtcfprrmMt-?? zX=XEP>aQ~!vi>##AuDq~ax~xczA5r?dH)|r{We8lRtP`<0uX=z1Rwwb2tWV=5P$## zAOHafKmY;|fB*y_009U<00I#BEd}O;KRPj%M*DQvkugJ=&(VukSbnu8L_GYB0eqe$ zf#$-1)v4jdp4AUq%~m?l$w88Grq){OaLv4B+$23Y5&N-`C+3x5RMo zqLmV_ssCkhkqrV6fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5P$##{y%|z zDGuKZzi{5ya`q@wz{ODuUS9FZyq4U%1iYd|!Ls?Z^Xr)ih1E&1a5g=A)n7 wF9=-eag#U-+gR5#^vzMHEcQ;T;f?u4NstTz5cquvY)UeBnC4wddo7;+KO`Y-761SM delta 710 zcmV;%0y+Kh?FWkue;m|jYcr6{E!c#`zJ{UM1z ziE(!c^ZCGcXLjD3o$tVW9}91HW}=!-ew%bxIXF5p(p&~>f0iTd^`Rr9qeq6uj*gXw zlk!k`bZj^&mb;u+Q)97yVWC(|7Sgo-vVN~>2iRnm;>$pi42Q*XAxYvGms{PT*oxTi z*T=t{iIGcIR!;w$YxaxpJ6${bZEt_>XCd{8r<(hM18*y->$m4 z-uRPWGv>S(;{6c2Lwpe8!w`Ey>`hufk=PgFqY(Q;99YNke+t)M{0jEG>#s8t8^^yx zuXA#3fBaee=Hm_Q4*&oF0000000000002Pm#Q)guNOJkoll6SQRBGMB`<-MkUriV5 zho{pgyDkQBT;H`cq0`yq^pZI*q829GC-~M## z)-B6%cxXP$G`25RshlwW{Z1RR@e;M=L0Fn`TuK)l5 diff --git a/test/images/volumes-tester/rbd/bootstrap.sh b/test/images/volumes-tester/rbd/bootstrap.sh index bb670aca05..a147b4c838 100755 --- a/test/images/volumes-tester/rbd/bootstrap.sh +++ b/test/images/volumes-tester/rbd/bootstrap.sh @@ -35,9 +35,29 @@ mkdir -p /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-1 sh ./osd.sh 0 sh ./osd.sh 1 -# Prepare a RBD volume -# NOTE: we need Ceph kernel modules on the host! -rbd import block foo +# Configure and start cephfs metadata server +sh ./mds.sh + +# Prepare a RBD volume "foo" (only with layering feature, the others may +# require newer clients). +# NOTE: we need Ceph kernel modules on the host that runs the client! +rbd import --image-feature layering block foo + +# Prepare a cephfs volume +ceph osd pool create cephfs_data 4 +ceph osd pool create cephfs_metadata 4 +ceph fs new cephfs cephfs_metadata cephfs_data +# Put index.html into the volume +# It takes a while until the volume created above is mountable, +# 1 second is usually enough, but try indefinetily. +sleep 1 +while ! ceph-fuse -m `hostname -i`:6789 /mnt; do + echo "Waiting for cephfs to be up" + sleep 1 +done +echo "Hello Ceph!" > /mnt/index.html +chmod 644 /mnt/index.html +umount /mnt echo "Ceph is ready" diff --git a/test/images/volumes-tester/rbd/ceph.conf.sh b/test/images/volumes-tester/rbd/ceph.conf.sh index ecafa43d6b..6ccfffda57 100755 --- a/test/images/volumes-tester/rbd/ceph.conf.sh +++ b/test/images/volumes-tester/rbd/ceph.conf.sh @@ -29,9 +29,13 @@ host = cephbox mon addr = $1 [osd] -osd journal size = 128 +osd journal size = 128 journal dio = false +# allow running on ext4 +osd max object name len = 256 +osd max object namespace len = 64 + [osd.0] osd host = cephbox " > /etc/ceph/ceph.conf diff --git a/test/images/volumes-tester/rbd/create_block.sh b/test/images/volumes-tester/rbd/create_block.sh index e865463a64..705957b1db 100755 --- a/test/images/volumes-tester/rbd/create_block.sh +++ b/test/images/volumes-tester/rbd/create_block.sh @@ -36,8 +36,9 @@ cleanup() trap cleanup TERM EXIT -# Create 1MB device with ext2 -dd if=/dev/zero of=block count=1 bs=1M +# Create 120MB device with ext2 +# (volume_io tests need at least 100MB) +dd if=/dev/zero of=block seek=120 count=1 bs=1M mkfs.ext2 block # Add index.html to it diff --git a/test/images/volumes-tester/rbd/mds.sh b/test/images/volumes-tester/rbd/mds.sh new file mode 100755 index 0000000000..744ccbf6b3 --- /dev/null +++ b/test/images/volumes-tester/rbd/mds.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Configures Ceph Metadata Service (mds), needed by CephFS +# +ceph-mds -i cephfs -c /etc/ceph/ceph.conf diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index dae027c326..d3ed428ebd 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -97,7 +97,7 @@ var ( VolumeISCSIServer = ImageConfig{gcRegistry, "volume-iscsi", "0.2", false} VolumeGlusterServer = ImageConfig{gcRegistry, "volume-gluster", "0.2", false} VolumeCephServer = ImageConfig{gcRegistry, "volume-ceph", "0.1", false} - VolumeRBDServer = ImageConfig{gcRegistry, "volume-rbd", "0.1", false} + VolumeRBDServer = ImageConfig{gcRegistry, "volume-rbd", "0.2", false} ) func GetE2EImage(image ImageConfig) string {