fixup! fixup! fixup! doc: document layers-scm

pym/bob/cmds/build/build.py

@@ -227,7 +227,7 @@ def _downloadLayerArgument(arg):
         recipes.setConfigFiles(args.configFile)
         if args.build_mode != 'build-only':
             setVerbosity(args.verbose)
-            updateLayers(recipes, loop, defines, args.verbose)
+            updateLayers(recipes, loop, defines, args.verbose, args.attic)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue

pym/bob/cmds/layers.py

@@ -16,6 +16,12 @@ def doLayers(argv, bobRoot):
         help="Increase verbosity (may be specified multiple times)")
     parser.add_argument('-D', default=[], action='append', dest="defines",
         help="Override default environment variable")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--attic', action='store_true', default=True,
+        help="Move scm to attic if inline switch is not possible (default).")
+    group.add_argument('--no-attic', action='store_false', default=None, dest='attic',
+        help="Do not move to attic, instead fail the build.")
+
     args = parser.parse_args(argv)
 
     setVerbosity(args.verbose)
@@ -26,11 +32,11 @@ def doLayers(argv, bobRoot):
         recipes = RecipeSet()
         recipes.setConfigFiles(args.configFile)
         if args.action == "update":
-            updateLayers(recipes, loop, defines, args.verbose)
+            updateLayers(recipes, loop, defines, args.verbose, args.attic)
 
         recipes.parse(defines)
 
-        layers = Layers(recipes, loop)
+        layers = Layers(recipes, loop, defines, args.attic)
         layers.collect(False, args.verbose)
         if args.action == "status":
             pp = PackagePrinter(args.verbose, False, False)

pym/bob/input.py

@@ -2953,7 +2953,7 @@ class RecipeSet:
         'import' : ImportScm.SCHEMA,
     })
 
-    STATIC_CONFIG_SCHEMA = schema.Schema({
+    STATIC_CONFIG_SCHEMA_SPEC = {
         schema.Optional('bobMinimumVersion') : str, # validated separately in preValidate
         schema.Optional('plugins') : [str],
         schema.Optional('policies') : schema.Schema(
@@ -2982,12 +2982,45 @@ class RecipeSet:
         schema.Optional('scriptLanguage',
                         default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
                                                                   schema.Use(ScriptLanguage)),
-    })
+    }
 
     MIRRORS_SCHEMA = ScmValidator({
         'url' : UrlScm.MIRRORS_SCHEMA,
     })
 
+    POLICIES = {
+        'noUndefinedTools' : (
+            "0.17.3.dev57",
+            InfoOnce("noUndefinedTools policy not set. Included but undefined tools are not detected at parsing time.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#noundefinedtools for more information.")
+        ),
+        'scmIgnoreUser' : (
+            "0.17.3.dev97",
+            InfoOnce("scmIgnoreUser policy not set. Authentication part URL is tainting binary artifacts.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#scmignoreuser for more information.")
+        ),
+        'pruneImportScm' : (
+            "0.17.3.dev102",
+            InfoOnce("pruneImportScm policy not set. Incremental builds of 'import' SCM may lead to wrong results.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#pruneimportscm for more information.")
+        ),
+        'gitCommitOnBranch' : (
+            "0.21.0.dev5",
+            InfoOnce("gitCommitOnBranch policy not set. Will not check if commit / tag is on configured branch.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#gitcommitonbranch for more information.")
+        ),
+        'fixImportScmVariant' : (
+            "0.22.1.dev34",
+            InfoOnce("fixImportScmVariant policy not set. Recipe variant calculation w/ import SCM is boguous.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#fiximportscmvariant for more information.")
+        ),
+        "defaultFileMode": (
+            "0.24rc1",
+            InfoOnce("defaultFileMode policy not set. File mode of URL SCMs not set for locally copied files.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#defaultfilemode for more information.")
+        ),
+    }
+
     _ignoreCmdConfig = False
     @classmethod
     def ignoreCommandCfg(cls):
@@ -3023,38 +3056,7 @@ def __init__(self):
         self.__uiConfig = {}
         self.__shareConfig = {}
         self.__layers = []
-        self.__policies = {
-            'noUndefinedTools' : (
-                "0.17.3.dev57",
-                InfoOnce("noUndefinedTools policy not set. Included but undefined tools are not detected at parsing time.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#noundefinedtools for more information.")
-            ),
-            'scmIgnoreUser' : (
-                "0.17.3.dev97",
-                InfoOnce("scmIgnoreUser policy not set. Authentication part URL is tainting binary artifacts.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#scmignoreuser for more information.")
-            ),
-            'pruneImportScm' : (
-                "0.17.3.dev102",
-                InfoOnce("pruneImportScm policy not set. Incremental builds of 'import' SCM may lead to wrong results.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#pruneimportscm for more information.")
-            ),
-            'gitCommitOnBranch' : (
-                "0.21.0.dev5",
-                InfoOnce("gitCommitOnBranch policy not set. Will not check if commit / tag is on configured branch.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#gitcommitonbranch for more information.")
-            ),
-            'fixImportScmVariant' : (
-                "0.22.1.dev34",
-                InfoOnce("fixImportScmVariant policy not set. Recipe variant calculation w/ import SCM is boguous.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#fiximportscmvariant for more information.")
-            ),
-            "defaultFileMode": (
-                "0.24rc1",
-                InfoOnce("defaultFileMode policy not set. File mode of URL SCMs not set for locally copied files.",
-                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#defaultfilemode for more information.")
-            ),
-        }
+        self.__policies = RecipeSet.POLICIES.copy()
         self.__buildHooks = {}
         self.__sandboxOpts = {}
         self.__scmDefaults = {}
@@ -3089,6 +3091,11 @@ def removeWhiteList(x):
 
         archiveValidator = ArchiveValidator()
 
+        self.__whitelistSetting = BuiltinSetting(
+                schema.Schema([ schema.Regex(r'^[^=]*$') ]),
+                updateWhiteList
+            )
+
         self.__settings = {
             "include" : SentinelSetting(),
             "require" : SentinelSetting(),
@@ -3180,10 +3187,7 @@ def removeWhiteList(x):
                 }),
                 lambda x: updateDicRecursive(self.__uiConfig, x)
             ),
-            "whitelist" : BuiltinSetting(
-                schema.Schema([ schema.Regex(r'^[^=]*$') ]),
-                updateWhiteList
-            ),
+            "whitelist" : self.__whitelistSetting,
             "whitelistRemove" : BuiltinSetting(
                 schema.Schema([ schema.Regex(r'^[^=]*$') ]),
                 removeWhiteList,
@@ -3437,6 +3441,7 @@ def __parse(self, envOverrides, platform, recipesRoot="", noLayers=False):
             raise ParseError("Invalid platform: " + platform)
         self.__platform = platform
         self.__layers = []
+        self.__policies = RecipeSet.POLICIES.copy()
         self.__whiteList = set()
         if platform == 'win32':
             self.__whiteList |= set(["ALLUSERSPROFILE", "APPDATA",
@@ -3464,19 +3469,11 @@ def __parse(self, envOverrides, platform, recipesRoot="", noLayers=False):
             self.__parseUserConfig(os.path.join(os.environ.get('XDG_CONFIG_HOME',
                 os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml'))
 
-        osEnv = Env(os.environ)
-        osEnv.setFuns(self.__stringFunctions)
-        env = Env({ k : osEnv.substitute(v, k) for (k, v) in
-            self.__defaultEnv.items() })
-        env.setFuns(self.__stringFunctions)
-        env.update(envOverrides)
-        env["BOB_HOST_PLATFORM"] = platform
-        self.__rootEnv = env
+        # Begin with root layer
+        self.__parseLayer(LayerSpec(""), "9999", recipesRoot, noLayers)
 
         if noLayers:
             return
-        # Begin with root layer
-        self.__parseLayer(LayerSpec(""), "9999", recipesRoot)
 
         # Out-of-tree builds may have a dedicated default.yaml
         if recipesRoot:
@@ -3490,6 +3487,7 @@ def __parse(self, envOverrides, platform, recipesRoot="", noLayers=False):
             self.__parseUserConfig(c)
 
         # calculate start environment
+        osEnv = Env(os.environ)
         osEnv.setFuns(self.__stringFunctions)
         env = Env({ k : osEnv.substitute(v, k) for (k, v) in
             self.__defaultEnv.items() })
@@ -3518,7 +3516,7 @@ def __parse(self, envOverrides, platform, recipesRoot="", noLayers=False):
         self.__rootRecipe = Recipe.createVirtualRoot(self, sorted(filteredRoots), self.__properties)
         self.__addRecipe(self.__rootRecipe)
 
-    def __parseLayer(self, layerSpec, maxVer, recipesRoot):
+    def __parseLayer(self, layerSpec, maxVer, recipesRoot, noLayers=False):
         layer = layerSpec.getName()
 
         if layer in self.__layers:
@@ -3542,8 +3540,17 @@ def preValidate(data):
             if compareVersion(BOB_VERSION, minVer) < 0:
                 raise ParseError("Your Bob is too old. At least version "+minVer+" is required!")
 
-        config = self.loadYaml(configYaml, (RecipeSet.STATIC_CONFIG_SCHEMA, b''),
+        config_spec = RecipeSet.STATIC_CONFIG_SCHEMA_SPEC
+        config_spec[schema.Optional("whitelist")] = self.__whitelistSetting.validate
+
+        config = self.loadYaml(configYaml, (schema.Schema(config_spec), b''),
             preValidate=preValidate)
+
+        # merge settings
+        for (name, value) in sorted([s for s in config.items() if s[0] in self.__settings],
+                                    key=lambda i: self.__settings[i[0]].priority):
+            self.__settings[name].merge(value)
+
         minVer = config.get("bobMinimumVersion", "0.16")
         if compareVersion(maxVer, minVer) < 0:
             raise ParseError("Layer '{}' requires a higher Bob version than root project!"
@@ -3567,6 +3574,9 @@ def preValidate(data):
             for (name, behaviour) in config.get("policies", {}).items():
                 self.__policies[name] = (behaviour, None)
 
+        if noLayers:
+            return
+
         # First parse any sub-layers. Their settings have a lower precedence
         # and may be overwritten by higher layers.
         for l in config.get("layers", []):