diff --git a/osgtest/tests/test_150_xrootd.py b/osgtest/tests/test_150_xrootd.py index 7c376652..3d115584 100644 --- a/osgtest/tests/test_150_xrootd.py +++ b/osgtest/tests/test_150_xrootd.py @@ -31,6 +31,10 @@ class TestStartXrootd(osgunittest.OSGTestCase): + def setUp(self): + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") + def test_01_start_xrootd(self): core.config['xrootd.pid-file'] = '/var/run/xrootd/xrootd-default.pid' core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem' diff --git a/osgtest/tests/test_155_stashcache.py b/osgtest/tests/test_155_stashcache.py index 77258ae6..0d06631e 100644 --- a/osgtest/tests/test_155_stashcache.py +++ b/osgtest/tests/test_155_stashcache.py @@ -7,154 +7,178 @@ from osgtest.library import service -CACHE_DIR = "/tmp/sccache" -CACHE_XROOT_PORT = 1094 # can't change this - stashcp doesn't allow you to specify port -CACHE_HTTP_PORT = 8001 -ORIGIN_XROOT_PORT = 1095 -ORIGIN_DIR = "/tmp/scorigin" -CACHE_AUTHFILE_PATH = "/etc/xrootd/Authfile-cache" -CACHE_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-cache-server.cfg" -ORIGIN_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-origin-server.cfg" -CACHES_JSON_PATH = "/etc/stashcache/caches.json" - - -# TODO Set up authenticated stashcache as well -CACHE_CONFIG_TEXT = """\ -all.export / -set cachedir = {CACHE_DIR} -xrd.allow host * -sec.protocol host -all.adminpath /var/spool/xrootd - -xrootd.trace emsg login stall redirect -ofs.trace all -xrd.trace all -cms.trace all - -ofs.osslib libXrdPss.so -# normally this is the redirector but we don't have one in this environment -pss.origin localhost:{ORIGIN_XROOT_PORT} -pss.cachelib libXrdFileCache.so -pss.setopt DebugLevel 1 - -oss.localroot $(cachedir) - -pfc.blocksize 512k -pfc.ram 1024m -# ^ xrootd won't start without a gig -pfc.prefetch 10 -pfc.diskusage 0.90 0.95 - -ofs.authorize 1 -acc.audit deny grant - -acc.authdb {CACHE_AUTHFILE_PATH} -sec.protbind * none -xrd.protocol http:{CACHE_HTTP_PORT} libXrdHttp.so - -xrd.port {CACHE_XROOT_PORT} - -http.listingdeny yes -http.staticpreload http://static/robots.txt /etc/xrootd/stashcache-robots.txt - -# Tune the client timeouts to more aggressively timeout. -pss.setopt ParallelEvtLoop 10 -pss.setopt RequestTimeout 25 -#pss.setopt TimeoutResolution 1 -pss.setopt ConnectTimeout 25 -pss.setopt ConnectionRetry 2 -#pss.setopt StreamTimeout 35 - -all.sitename osgtest - -xrootd.diglib * /etc/xrootd/digauth.cf -""".format(**globals()) - - -CACHE_AUTHFILE_TEXT = """\ -u * / rl +# These will end up as environment variables in the xrootd configs +# as well core.config["stashcache.KEY"] = var +# Xrootd config syntax doesn't allow underscores so this is CamelCase. +PARAMS = dict( + CacheRootdir = "/tmp/sccache", + CacheXrootPort = 1094, # can't change this - stashcp doesn't allow you to specify port + CacheHTTPPort = 8001, + CacheHTTPSPort = 8444, + OriginXrootPort = 1095, + OriginAuthXrootPort = 1096, + OriginRootdir = "/tmp/scorigin", + OriginExport = "/osgtest/PUBLIC", + OriginAuthExport = "/osgtest/PROTECTED", + OriginDummyExport = "/osgtest/dummy", + # ^ originexport needs to be defined on caches too because they use the same config.d + # This is relative to CacheRootdir, not OriginRootdir + StashOriginAuthfile = "/etc/xrootd/Authfile-origin", + StashOriginPublicAuthfile = "/etc/xrootd/Authfile-origin-public", + StashCacheAuthfile = "/etc/xrootd/Authfile-cache", + StashCachePublicAuthfile = "/etc/xrootd/Authfile-cache-public", + OriginResourcename = "OSG_TEST_ORIGIN", + CacheResourcename = "OSG_TEST_CACHE", +) + +PARAMS_CFG_PATH = "/etc/xrootd/config.d/01-params.cfg" +# Some statements can take env vars on the right hand side (setenv); others take config vars (set) +# so define both. +PARAMS_CFG_CONTENTS = "\n".join("setenv {0} = {1}\nset {0} = {1}".format(k, v) + for k, v in PARAMS.items()) + "\n" + +PRE_CFG_PATH = "/etc/xrootd/config.d/11-pre.cfg" +PRE_CFG_CONTENTS = """ +set DisableOsgMonitoring = 1 + +if named stash-cache-auth + xrd.port $(CacheHTTPSPort) + set rootdir = $(CacheRootdir) + set resourcename = $(CacheResourcename) + set originexport = $(OriginDummyExport) + + ofs.osslib libXrdPss.so + pss.cachelib libXrdFileCache.so + + pss.origin localhost:$(OriginAuthXrootPort) + xrd.protocol http:$(CacheHTTPSPort) libXrdHttp.so +else if named stash-cache + xrd.port $(CacheXrootPort) + set rootdir = $(CacheRootdir) + set resourcename = $(CacheResourcename) + set originexport = $(OriginDummyExport) + + ofs.osslib libXrdPss.so + pss.cachelib libXrdFileCache.so + + pss.origin localhost:$(OriginXrootPort) + xrd.protocol http:$(CacheHTTPPort) libXrdHttp.so +else if named stash-origin-auth + xrd.port $(OriginAuthXrootPort) + set rootdir = $(OriginRootdir) + set resourcename = $(OriginResourcename) + set originexport = $(OriginAuthExport) +else if named stash-origin + xrd.port $(OriginXrootPort) + set rootdir = $(OriginRootdir) + set resourcename = $(OriginResourcename) + set originexport = $(OriginExport) +fi """ +CACHE_AUTHFILE_PATH = PARAMS["StashCacheAuthfile"] +CACHE_AUTHFILE_CONTENTS = "u * / rl\n" -ORIGIN_CONFIG_TEXT = """\ -xrd.allow host * -sec.protocol host -sec.protbind * none -all.adminpath /var/spool/xrootd -all.pidpath /var/run/xrootd - -# The directory on local disk containing the files to share, e.g. "/stash". -oss.localroot {ORIGIN_DIR} -all.export / - -xrd.port {ORIGIN_XROOT_PORT} -all.role server +CACHE_PUBLIC_AUTHFILE_PATH = PARAMS["StashCachePublicAuthfile"] +CACHE_PUBLIC_AUTHFILE_CONTENTS = "u * / rl\n" -xrootd.trace emsg login stall redirect -ofs.trace all -xrd.trace all -cms.trace all -""".format(**globals()) +ORIGIN_AUTHFILE_PATH = PARAMS["StashOriginAuthfile"] +ORIGIN_AUTHFILE_CONTENTS = "u * /osgtest/PROTECTED rl\n" +ORIGIN_PUBLIC_AUTHFILE_PATH = PARAMS["StashOriginPublicAuthfile"] +ORIGIN_PUBLIC_AUTHFILE_CONTENTS = "u * /osgtest/PUBLIC rl\n" -CACHES_JSON_TEXT = """\ +CACHES_JSON_PATH = "/etc/stashcache/caches.json" +CACHES_JSON_CONTENTS = """\ [ {"name":"root://localhost", "status":1, "longitude":-89.4012, "latitude":43.0731} ] """ +XROOTD_ORIGIN_CFG_PATH = "/etc/xrootd/xrootd-stash-origin.cfg" +HTTP_CFG_PATH = "/etc/xrootd/config.d/40-osg-http.cfg" +CACHING_PLUGIN_CFG_PATH = "/etc/xrootd/config.d/40-osg-caching-plugin.cfg" + -_NAMESPACE = "stashcache" +NAMESPACE = "stashcache" -def _getcfg(key): - return core.config["%s.%s" % (_NAMESPACE, key)] +def setcfg(key, val): + core.config["%s.%s" % (NAMESPACE, key)] = val -def _setcfg(key, val): - core.config["%s.%s" % (_NAMESPACE, key)] = val +def start_xrootd(instance): + svc = "xrootd@%s" % instance + if not service.is_running(svc): + service.check_start(svc) class TestStartStashCache(OSGTestCase): @core.elrelease(7,8) def setUp(self): - core.skip_ok_unless_installed("stashcache-origin-server", - "stashcache-cache-server", + core.skip_ok_unless_installed("stash-origin", + "stash-cache", "stashcache-client", by_dependency=True) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") < "1.0.2", "needs xcache 1.0.2+") def test_01_configure(self): - for key, val in [ - ("cache_authfile_path", CACHE_AUTHFILE_PATH), - ("cache_config_path", CACHE_CONFIG_PATH), - ("origin_config_path", ORIGIN_CONFIG_PATH), - ("caches_json_path", CACHES_JSON_PATH), - ("cache_http_port", CACHE_HTTP_PORT), - ("origin_dir", ORIGIN_DIR), - ("cache_dir", CACHE_DIR), - ("origin_xroot_port", ORIGIN_XROOT_PORT), - ("cache_xroot_port", CACHE_XROOT_PORT) - ]: - _setcfg(key, val) - - xrootd_user = pwd.getpwnam("xrootd") - for d in [_getcfg("origin_dir"), _getcfg("cache_dir"), - os.path.dirname(_getcfg("caches_json_path"))]: + for key, val in PARAMS.items(): + setcfg(key, val) + + # Create dirs + for d in [PARAMS["OriginRootdir"], + PARAMS["CacheRootdir"], + os.path.join(PARAMS["OriginRootdir"], PARAMS["OriginExport"].lstrip("/")), + os.path.join(PARAMS["OriginRootdir"], PARAMS["OriginAuthExport"].lstrip("/")), + os.path.join(PARAMS["CacheRootdir"], PARAMS["OriginDummyExport"].lstrip("/")), + os.path.dirname(CACHES_JSON_PATH)]: files.safe_makedirs(d) - os.chown(d, xrootd_user.pw_uid, xrootd_user.pw_gid) - for key, text in [ - ("cache_config_path", CACHE_CONFIG_TEXT), - ("cache_authfile_path", CACHE_AUTHFILE_TEXT), - ("origin_config_path", ORIGIN_CONFIG_TEXT), - ("caches_json_path", CACHES_JSON_TEXT) + core.system(["chown", "-R", "xrootd:xrootd", PARAMS["OriginRootdir"], PARAMS["CacheRootdir"]]) + + filelist = [] + setcfg("filelist", filelist) + # Modify filelist in-place with .append so changes get into core.config too + + # Delete the lines we can't override + for path, regexp in [ + (XROOTD_ORIGIN_CFG_PATH, "^\s*all.manager.+$"), + (HTTP_CFG_PATH, "^\s*xrd.protocol.+$"), + (CACHING_PLUGIN_CFG_PATH, "^\s*(ofs.osslib|pss.cachelib|pss.origin).+$"), + ]: + files.replace_regexpr(path, regexp, "", owner=NAMESPACE) + filelist.append(path) + + # Write our new files + for path, contents in [ + (PARAMS_CFG_PATH, PARAMS_CFG_CONTENTS), + (PRE_CFG_PATH, PRE_CFG_CONTENTS), + (ORIGIN_AUTHFILE_PATH, ORIGIN_AUTHFILE_CONTENTS), + (ORIGIN_PUBLIC_AUTHFILE_PATH, ORIGIN_PUBLIC_AUTHFILE_CONTENTS), + (CACHE_AUTHFILE_PATH, CACHE_AUTHFILE_CONTENTS), + (CACHE_PUBLIC_AUTHFILE_PATH, CACHE_PUBLIC_AUTHFILE_CONTENTS), + (CACHES_JSON_PATH, CACHES_JSON_CONTENTS) ]: - files.write(_getcfg(key), text, owner=_NAMESPACE, chmod=0o644) + files.write(path, contents, owner=NAMESPACE, chmod=0o644) + filelist.append(path) + + # Install certs. Normally done in the xrootd tests but they conflict with the StashCache tests + # (both use the same config dir) + core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem' + core.config['certs.xrootdkey'] = '/etc/grid-security/xrd/xrdkey.pem' + core.install_cert('certs.xrootdcert', 'certs.hostcert', 'xrootd', 0o644) + core.install_cert('certs.xrootdkey', 'certs.hostkey', 'xrootd', 0o400) + + def test_02_start_stash_origin(self): + start_xrootd("stash-origin") + + def test_03_start_stash_origin_auth(self): + start_xrootd("stash-origin-auth") - def test_02_start_origin(self): - if not service.is_running("xrootd@stashcache-origin-server"): - service.check_start("xrootd@stashcache-origin-server") + def test_04_start_stash_cache(self): + start_xrootd("stash-cache") - def test_03_start_cache(self): - if not service.is_running("xrootd@stashcache-cache-server"): - service.check_start("xrootd@stashcache-cache-server") + def test_05_start_stash_cache_auth(self): + start_xrootd("stash-cache-auth") diff --git a/osgtest/tests/test_158_xrootd_tpc.py b/osgtest/tests/test_158_xrootd_tpc.py index 247e32c2..69ac5f9e 100644 --- a/osgtest/tests/test_158_xrootd_tpc.py +++ b/osgtest/tests/test_158_xrootd_tpc.py @@ -54,6 +54,8 @@ class TestStartXrootdTPC(osgunittest.OSGTestCase): def setUp(self): core.skip_ok_unless_installed("xrootd", by_dependency=True) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") def test_01_configure_xrootd(self): core.config['xrootd.tpc.config-1'] = '/etc/xrootd/xrootd-third-party-copy-1.cfg' diff --git a/osgtest/tests/test_450_xrootd.py b/osgtest/tests/test_450_xrootd.py index 511bb30f..df17b29d 100644 --- a/osgtest/tests/test_450_xrootd.py +++ b/osgtest/tests/test_450_xrootd.py @@ -14,6 +14,10 @@ class TestXrootd(osgunittest.OSGTestCase): __data_path = '/usr/share/osg-test/test_gridftp_data.txt' __fuse_path = '/mnt/xrootd_fuse_test' + def setUp(self): + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") + def test_01_xrdcp_local_to_server(self): core.state['xrootd.copied-to-server'] = False core.skip_ok_unless_installed('xrootd', 'xrootd-client', by_dependency=True) diff --git a/osgtest/tests/test_460_stashcache.py b/osgtest/tests/test_460_stashcache.py index 120ad0a7..24c19522 100644 --- a/osgtest/tests/test_460_stashcache.py +++ b/osgtest/tests/test_460_stashcache.py @@ -13,24 +13,22 @@ from urllib.request import urlopen -_NAMESPACE = "stashcache" +NAMESPACE = "stashcache" -def _getcfg(key): - return core.config["%s.%s" % (_NAMESPACE, key)] - -def _setcfg(key, val): - core.config["%s.%s" % (_NAMESPACE, key)] = val +def getcfg(key): + return core.config["%s.%s" % (NAMESPACE, key)] +# TODO Work with authed origin/cache as well. A separate class would probably be the best. class TestStashCache(OSGTestCase): # testfiles with random contents testfiles = [ - ("testfile%d" % x, str(random.random())) + ("testfile%d" % x, str(random.random()) + "\n") for x in range(4) ] def assertCached(self, name, contents): - fpath = os.path.join(_getcfg("cache_dir"), name) + fpath = os.path.join(getcfg("CacheRootdir"), getcfg("OriginExport").lstrip("/"), name) self.assertTrue(os.path.exists(fpath), name + " not cached") self.assertEqualVerbose(actual=files.read(fpath, as_single_string=True), @@ -43,38 +41,36 @@ def skip_bad_unless_running(self, *services): @core.elrelease(7,8) def setUp(self): - core.skip_ok_unless_installed("stashcache-origin-server", - "stashcache-cache-server", + core.skip_ok_unless_installed("stash-origin", + "stash-cache", "stashcache-client", by_dependency=True) - if core.rpm_is_installed('xcache'): - origin_service = "xrootd@stash-origin" - cache_service = "xrootd@stash-cache" - else: - origin_service = "xrootd@stashcache-origin-server" - cache_service = "xrootd@stashcache-cache-server" - self.skip_bad_unless_running(origin_service, cache_service) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") < "1.0.2", "needs xcache 1.0.2+") + self.skip_bad_unless_running("xrootd@stash-origin", "xrootd@stash-cache") def test_01_create_files(self): xrootd_user = pwd.getpwnam("xrootd") for name, contents in self.testfiles: - files.write(os.path.join(_getcfg("origin_dir"), name), + files.write(os.path.join(getcfg("OriginRootdir"), getcfg("OriginExport").lstrip("/"), name), contents, backup=False, chmod=0o644, chown=(xrootd_user.pw_uid, xrootd_user.pw_gid)) def test_02_xroot_fetch_from_origin(self): name, contents = self.testfiles[0] + path = os.path.join(getcfg("OriginExport"), name) result, _, _ = \ core.check_system(["xrdcp", "-d1", "-N", "-f", - "root://localhost:%d//%s" % (_getcfg("origin_xroot_port"), name), + "root://localhost:%d/%s" % (getcfg("OriginXrootPort"), path), "-"], "Checking xroot copy from origin") self.assertEqualVerbose(result, contents, "downloaded file mismatch") def test_03_http_fetch_from_cache(self): name, contents = self.testfiles[1] + path = os.path.join(getcfg("OriginExport"), name) try: f = urlopen( - "http://localhost:%d/%s" % (_getcfg("cache_http_port"), name) + "http://localhost:%d/%s" % (getcfg("CacheHTTPPort"), path) ) result = f.read() except IOError as e: @@ -84,9 +80,10 @@ def test_03_http_fetch_from_cache(self): def test_04_xroot_fetch_from_cache(self): name, contents = self.testfiles[2] + path = os.path.join(getcfg("OriginExport"), name) result, _, _ = \ core.check_system(["xrdcp", "-d1", "-N", "-f", - "root://localhost:%d//%s" % (_getcfg("cache_xroot_port"), name), + "root://localhost:%d/%s" % (getcfg("CacheXrootPort"), path), "-"], "Checking xroot copy from cache") self.assertEqualVerbose(result, contents, "downloaded file mismatch") self.assertCached(name, contents) @@ -96,8 +93,9 @@ def test_05_stashcp(self): if core.PackageVersion('stashcache-client') < '5.1.0-5': command.append("--cache=root://localhost") name, contents = self.testfiles[3] + path = os.path.join(getcfg("OriginExport"), name) with tempfile.NamedTemporaryFile(mode="r+b") as tf: - core.check_system(command + ["/"+name, tf.name], + core.check_system(command + [path, tf.name], "Checking stashcp") result = tf.read() self.assertEqualVerbose(result, contents, "stashcp'ed file mismatch") diff --git a/osgtest/tests/test_465_xrootd_tpc.py b/osgtest/tests/test_465_xrootd_tpc.py index fc2b5423..ec125df6 100644 --- a/osgtest/tests/test_465_xrootd_tpc.py +++ b/osgtest/tests/test_465_xrootd_tpc.py @@ -12,6 +12,8 @@ class TestXrootdTPC(osgunittest.OSGTestCase): def setUp(self): core.skip_ok_unless_installed("xrootd", by_dependency=True) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") def test_01_create_macaroons(self): core.skip_ok_unless_installed('x509-scitokens-issuer-client', by_dependency=True) diff --git a/osgtest/tests/test_835_stashcache.py b/osgtest/tests/test_835_stashcache.py index 91055a3b..e43d0cbe 100644 --- a/osgtest/tests/test_835_stashcache.py +++ b/osgtest/tests/test_835_stashcache.py @@ -6,41 +6,50 @@ from osgtest.library import service -_NAMESPACE = "stashcache" +NAMESPACE = "stashcache" -def _getcfg(key): - return core.config["%s.%s" % (_NAMESPACE, key)] +def getcfg(key): + return core.config["%s.%s" % (NAMESPACE, key)] -def _setcfg(key, val): - core.config["%s.%s" % (_NAMESPACE, key)] = val +def stop_xrootd(instance): + svc = "xrootd@%s" % instance + service.check_stop(svc) class TestStopStashCache(OSGTestCase): @core.elrelease(7,8) def setUp(self): - core.skip_ok_unless_installed("stashcache-origin-server", - "stashcache-cache-server", + core.skip_ok_unless_installed("stash-origin", + "stash-cache", "stashcache-client", by_dependency=True) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") < "1.0.2", "needs xcache 1.0.2+") - def test_01_stop_origin(self): - service.check_stop("xrootd@stashcache-origin-server") - - def test_02_stop_cache(self): - service.check_stop("xrootd@stashcache-cache-server") - - def test_03_unconfigure(self): - for key in [ - "cache_config_path", - "cache_authfile_path", - "origin_config_path", - "caches_json_path" - ]: - files.restore(_getcfg(key), owner=_NAMESPACE) - - def test_04_delete_dirs(self): - for key in ["origin_dir", "cache_dir"]: - if os.path.isdir(_getcfg(key)): - shutil.rmtree(_getcfg(key)) + def test_01_stop_stash_origin(self): + stop_xrootd("stash-origin") + + def test_02_stop_stash_origin_auth(self): + stop_xrootd("stash-origin-auth") + + def test_03_stop_stash_cache(self): + stop_xrootd("stash-cache") + + def test_04_stop_stash_cache_auth(self): + stop_xrootd("stash-cache-auth") + + def test_05_unconfigure(self): + for path in getcfg("filelist"): + files.restore(path, owner=NAMESPACE) + + def test_06_delete_dirs(self): + for key in ["OriginRootdir", "CacheRootdir"]: + if os.path.isdir(getcfg(key)): + shutil.rmtree(getcfg(key)) + + def test_07_remove_certs(self): + # Do the keys first, so that the directories will be empty for the certs. + core.remove_cert('certs.xrootdkey') + core.remove_cert('certs.xrootdcert') diff --git a/osgtest/tests/test_838_xrootd_tpc.py b/osgtest/tests/test_838_xrootd_tpc.py index c73f09a4..4c7e6696 100644 --- a/osgtest/tests/test_838_xrootd_tpc.py +++ b/osgtest/tests/test_838_xrootd_tpc.py @@ -8,6 +8,8 @@ class TestStopXrootdTPC(osgunittest.OSGTestCase): def setUp(self): core.skip_ok_unless_installed("xrootd", by_dependency=True) + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") def test_01_stop_xrootd(self): if core.state['xrootd.tpc.backups-exist']: diff --git a/osgtest/tests/test_840_xrootd.py b/osgtest/tests/test_840_xrootd.py index c3720535..751adbe3 100644 --- a/osgtest/tests/test_840_xrootd.py +++ b/osgtest/tests/test_840_xrootd.py @@ -4,6 +4,9 @@ import osgtest.library.osgunittest as osgunittest class TestStopXrootd(osgunittest.OSGTestCase): + def setUp(self): + if core.rpm_is_installed("xcache"): + self.skip_ok_if(core.PackageVersion("xcache") >= "1.0.2", "xcache 1.0.2+ configs conflict with xrootd tests") def test_01_stop_xrootd(self): if core.state['xrootd.backups-exist']: