Skip to content

Commit

Permalink
Merge pull request #168 from opensciencegrid/xcache-1.0.2
Browse files Browse the repository at this point in the history 
Update tests to work with XCache 1.0.2+
  • Loading branch information
@brianhlin
brianhlin authored May 3, 2019
2 parents f9d923e + 69d7f60 commit bcaf9a7
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 169 deletions.
4 changes: 4 additions & 0 deletions osgtest/tests/test_150_xrootd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@

class TestStartXrootd(osgunittest.OSGTestCase):

def setUp(self):
if core.rpm_is_installed("xcache"):
self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests")

def test_01_start_xrootd(self):
core.config['xrootd.pid-file'] = '/var/run/xrootd/xrootd-default.pid'
core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem'
Expand Down
266 changes: 145 additions & 121 deletions osgtest/tests/test_155_stashcache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,154 +7,178 @@
from osgtest.library import service


CACHE_DIR = "/tmp/sccache"
CACHE_XROOT_PORT = 1094 # can't change this - stashcp doesn't allow you to specify port
CACHE_HTTP_PORT = 8001
ORIGIN_XROOT_PORT = 1095
ORIGIN_DIR = "/tmp/scorigin"
CACHE_AUTHFILE_PATH = "/etc/xrootd/Authfile-cache"
CACHE_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-cache-server.cfg"
ORIGIN_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-origin-server.cfg"
CACHES_JSON_PATH = "/etc/stashcache/caches.json"


# TODO Set up authenticated stashcache as well
CACHE_CONFIG_TEXT = """\
all.export /
set cachedir = {CACHE_DIR}
xrd.allow host *
sec.protocol host
all.adminpath /var/spool/xrootd
xrootd.trace emsg login stall redirect
ofs.trace all
xrd.trace all
cms.trace all
ofs.osslib libXrdPss.so
# normally this is the redirector but we don't have one in this environment
pss.origin localhost:{ORIGIN_XROOT_PORT}
pss.cachelib libXrdFileCache.so
pss.setopt DebugLevel 1
oss.localroot $(cachedir)
pfc.blocksize 512k
pfc.ram 1024m
# ^ xrootd won't start without a gig
pfc.prefetch 10
pfc.diskusage 0.90 0.95
ofs.authorize 1
acc.audit deny grant
acc.authdb {CACHE_AUTHFILE_PATH}
sec.protbind * none
xrd.protocol http:{CACHE_HTTP_PORT} libXrdHttp.so
xrd.port {CACHE_XROOT_PORT}
http.listingdeny yes
http.staticpreload http://static/robots.txt /etc/xrootd/stashcache-robots.txt
# Tune the client timeouts to more aggressively timeout.
pss.setopt ParallelEvtLoop 10
pss.setopt RequestTimeout 25
#pss.setopt TimeoutResolution 1
pss.setopt ConnectTimeout 25
pss.setopt ConnectionRetry 2
#pss.setopt StreamTimeout 35
all.sitename osgtest
xrootd.diglib * /etc/xrootd/digauth.cf
""".format(**globals())


CACHE_AUTHFILE_TEXT = """\
u * / rl
# These will end up as environment variables in the xrootd configs
# as well core.config["stashcache.KEY"] = var
# Xrootd config syntax doesn't allow underscores so this is CamelCase.
PARAMS = dict(
CacheRootdir = "/tmp/sccache",
CacheXrootPort = 1094, # can't change this - stashcp doesn't allow you to specify port
CacheHTTPPort = 8001,
CacheHTTPSPort = 8444,
OriginXrootPort = 1095,
OriginAuthXrootPort = 1096,
OriginRootdir = "/tmp/scorigin",
OriginExport = "/osgtest/PUBLIC",
OriginAuthExport = "/osgtest/PROTECTED",
OriginDummyExport = "/osgtest/dummy",
# ^ originexport needs to be defined on caches too because they use the same config.d
# This is relative to CacheRootdir, not OriginRootdir
StashOriginAuthfile = "/etc/xrootd/Authfile-origin",
StashOriginPublicAuthfile = "/etc/xrootd/Authfile-origin-public",
StashCacheAuthfile = "/etc/xrootd/Authfile-cache",
StashCachePublicAuthfile = "/etc/xrootd/Authfile-cache-public",
OriginResourcename = "OSG_TEST_ORIGIN",
CacheResourcename = "OSG_TEST_CACHE",
)

PARAMS_CFG_PATH = "/etc/xrootd/config.d/01-params.cfg"
# Some statements can take env vars on the right hand side (setenv); others take config vars (set)
# so define both.
PARAMS_CFG_CONTENTS = "\n".join("setenv {0} = {1}\nset {0} = {1}".format(k, v)
for k, v in PARAMS.items()) + "\n"

PRE_CFG_PATH = "/etc/xrootd/config.d/11-pre.cfg"
PRE_CFG_CONTENTS = """
set DisableOsgMonitoring = 1
if named stash-cache-auth
xrd.port $(CacheHTTPSPort)
set rootdir = $(CacheRootdir)
set resourcename = $(CacheResourcename)
set originexport = $(OriginDummyExport)
ofs.osslib libXrdPss.so
pss.cachelib libXrdFileCache.so
pss.origin localhost:$(OriginAuthXrootPort)
xrd.protocol http:$(CacheHTTPSPort) libXrdHttp.so
else if named stash-cache
xrd.port $(CacheXrootPort)
set rootdir = $(CacheRootdir)
set resourcename = $(CacheResourcename)
set originexport = $(OriginDummyExport)
ofs.osslib libXrdPss.so
pss.cachelib libXrdFileCache.so
pss.origin localhost:$(OriginXrootPort)
xrd.protocol http:$(CacheHTTPPort) libXrdHttp.so
else if named stash-origin-auth
xrd.port $(OriginAuthXrootPort)
set rootdir = $(OriginRootdir)
set resourcename = $(OriginResourcename)
set originexport = $(OriginAuthExport)
else if named stash-origin
xrd.port $(OriginXrootPort)
set rootdir = $(OriginRootdir)
set resourcename = $(OriginResourcename)
set originexport = $(OriginExport)
fi
"""

CACHE_AUTHFILE_PATH = PARAMS["StashCacheAuthfile"]
CACHE_AUTHFILE_CONTENTS = "u * / rl\n"

ORIGIN_CONFIG_TEXT = """\
xrd.allow host *
sec.protocol host
sec.protbind * none
all.adminpath /var/spool/xrootd
all.pidpath /var/run/xrootd
# The directory on local disk containing the files to share, e.g. "/stash".
oss.localroot {ORIGIN_DIR}
all.export /
xrd.port {ORIGIN_XROOT_PORT}
all.role server
CACHE_PUBLIC_AUTHFILE_PATH = PARAMS["StashCachePublicAuthfile"]
CACHE_PUBLIC_AUTHFILE_CONTENTS = "u * / rl\n"

xrootd.trace emsg login stall redirect
ofs.trace all
xrd.trace all
cms.trace all
""".format(**globals())
ORIGIN_AUTHFILE_PATH = PARAMS["StashOriginAuthfile"]
ORIGIN_AUTHFILE_CONTENTS = "u * /osgtest/PROTECTED rl\n"

ORIGIN_PUBLIC_AUTHFILE_PATH = PARAMS["StashOriginPublicAuthfile"]
ORIGIN_PUBLIC_AUTHFILE_CONTENTS = "u * /osgtest/PUBLIC rl\n"

CACHES_JSON_TEXT = """\
CACHES_JSON_PATH = "/etc/stashcache/caches.json"
CACHES_JSON_CONTENTS = """\
[
{"name":"root://localhost", "status":1, "longitude":-89.4012, "latitude":43.0731}
]
"""

XROOTD_ORIGIN_CFG_PATH = "/etc/xrootd/xrootd-stash-origin.cfg"
HTTP_CFG_PATH = "/etc/xrootd/config.d/40-osg-http.cfg"
CACHING_PLUGIN_CFG_PATH = "/etc/xrootd/config.d/40-osg-caching-plugin.cfg"


_NAMESPACE = "stashcache"
NAMESPACE = "stashcache"


def _getcfg(key):
return core.config["%s.%s" % (_NAMESPACE, key)]
def setcfg(key, val):
core.config["%s.%s" % (NAMESPACE, key)] = val


def _setcfg(key, val):
core.config["%s.%s" % (_NAMESPACE, key)] = val
def start_xrootd(instance):
svc = "xrootd@%s" % instance
if not service.is_running(svc):
service.check_start(svc)


class TestStartStashCache(OSGTestCase):
@core.elrelease(7,8)
def setUp(self):
core.skip_ok_unless_installed("stashcache-origin-server",
"stashcache-cache-server",
core.skip_ok_unless_installed("stash-origin",
"stash-cache",
"stashcache-client",
by_dependency=True)
if core.rpm_is_installed("xcache"):
self.skip_ok_if(core.PackageVersion("xcache") < "1.0.2", "needs xcache 1.0.2+")

def test_01_configure(self):
for key, val in [
("cache_authfile_path", CACHE_AUTHFILE_PATH),
("cache_config_path", CACHE_CONFIG_PATH),
("origin_config_path", ORIGIN_CONFIG_PATH),
("caches_json_path", CACHES_JSON_PATH),
("cache_http_port", CACHE_HTTP_PORT),
("origin_dir", ORIGIN_DIR),
("cache_dir", CACHE_DIR),
("origin_xroot_port", ORIGIN_XROOT_PORT),
("cache_xroot_port", CACHE_XROOT_PORT)
]:
_setcfg(key, val)

xrootd_user = pwd.getpwnam("xrootd")
for d in [_getcfg("origin_dir"), _getcfg("cache_dir"),
os.path.dirname(_getcfg("caches_json_path"))]:
for key, val in PARAMS.items():
setcfg(key, val)

# Create dirs
for d in [PARAMS["OriginRootdir"],
PARAMS["CacheRootdir"],
os.path.join(PARAMS["OriginRootdir"], PARAMS["OriginExport"].lstrip("/")),
os.path.join(PARAMS["OriginRootdir"], PARAMS["OriginAuthExport"].lstrip("/")),
os.path.join(PARAMS["CacheRootdir"], PARAMS["OriginDummyExport"].lstrip("/")),
os.path.dirname(CACHES_JSON_PATH)]:
files.safe_makedirs(d)
os.chown(d, xrootd_user.pw_uid, xrootd_user.pw_gid)

for key, text in [
("cache_config_path", CACHE_CONFIG_TEXT),
("cache_authfile_path", CACHE_AUTHFILE_TEXT),
("origin_config_path", ORIGIN_CONFIG_TEXT),
("caches_json_path", CACHES_JSON_TEXT)
core.system(["chown", "-R", "xrootd:xrootd", PARAMS["OriginRootdir"], PARAMS["CacheRootdir"]])

filelist = []
setcfg("filelist", filelist)
# Modify filelist in-place with .append so changes get into core.config too

# Delete the lines we can't override
for path, regexp in [
(XROOTD_ORIGIN_CFG_PATH, "^\s*all.manager.+$"),
(HTTP_CFG_PATH, "^\s*xrd.protocol.+$"),
(CACHING_PLUGIN_CFG_PATH, "^\s*(ofs.osslib|pss.cachelib|pss.origin).+$"),
]:
files.replace_regexpr(path, regexp, "", owner=NAMESPACE)
filelist.append(path)

# Write our new files
for path, contents in [
(PARAMS_CFG_PATH, PARAMS_CFG_CONTENTS),
(PRE_CFG_PATH, PRE_CFG_CONTENTS),
(ORIGIN_AUTHFILE_PATH, ORIGIN_AUTHFILE_CONTENTS),
(ORIGIN_PUBLIC_AUTHFILE_PATH, ORIGIN_PUBLIC_AUTHFILE_CONTENTS),
(CACHE_AUTHFILE_PATH, CACHE_AUTHFILE_CONTENTS),
(CACHE_PUBLIC_AUTHFILE_PATH, CACHE_PUBLIC_AUTHFILE_CONTENTS),
(CACHES_JSON_PATH, CACHES_JSON_CONTENTS)
]:
files.write(_getcfg(key), text, owner=_NAMESPACE, chmod=0o644)
files.write(path, contents, owner=NAMESPACE, chmod=0o644)
filelist.append(path)

# Install certs. Normally done in the xrootd tests but they conflict with the StashCache tests
# (both use the same config dir)
core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem'
core.config['certs.xrootdkey'] = '/etc/grid-security/xrd/xrdkey.pem'
core.install_cert('certs.xrootdcert', 'certs.hostcert', 'xrootd', 0o644)
core.install_cert('certs.xrootdkey', 'certs.hostkey', 'xrootd', 0o400)

def test_02_start_stash_origin(self):
start_xrootd("stash-origin")

def test_03_start_stash_origin_auth(self):
start_xrootd("stash-origin-auth")

def test_02_start_origin(self):
if not service.is_running("xrootd@stashcache-origin-server"):
service.check_start("xrootd@stashcache-origin-server")
def test_04_start_stash_cache(self):
start_xrootd("stash-cache")

def test_03_start_cache(self):
if not service.is_running("xrootd@stashcache-cache-server"):
service.check_start("xrootd@stashcache-cache-server")
def test_05_start_stash_cache_auth(self):
start_xrootd("stash-cache-auth")
2 changes: 2 additions & 0 deletions osgtest/tests/test_158_xrootd_tpc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class TestStartXrootdTPC(osgunittest.OSGTestCase):
def setUp(self):
core.skip_ok_unless_installed("xrootd",
by_dependency=True)
if core.rpm_is_installed("xcache"):
self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests")

def test_01_configure_xrootd(self):
core.config['xrootd.tpc.config-1'] = '/etc/xrootd/xrootd-third-party-copy-1.cfg'
Expand Down
4 changes: 4 additions & 0 deletions osgtest/tests/test_450_xrootd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ class TestXrootd(osgunittest.OSGTestCase):
__data_path = '/usr/share/osg-test/test_gridftp_data.txt'
__fuse_path = '/mnt/xrootd_fuse_test'

def setUp(self):
if core.rpm_is_installed("xcache"):
self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests")

def test_01_xrdcp_local_to_server(self):
core.state['xrootd.copied-to-server'] = False
core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True)
Expand Down
Loading

0 comments on commit bcaf9a7

Please sign in to comment.