Skip to content

Commit

Permalink
Implement --steps
Browse files Browse the repository at this point in the history
  • Loading branch information
@avillar
avillar committed Nov 27, 2023
1 parent 8cddebe commit 79d4881
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 47 deletions.
16 changes: 13 additions & 3 deletions ogc/bblocks/entrypoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,13 @@

parser.add_argument(
'--filter',
help='Filter by building block id or file. Sets --clean to false'
help='Filter by building block id or file. Forces --clean to false'
)

parser.add_argument(
'--steps',
help='Comma-separated list of postprocessing steps that will run (annotate,jsonld,superbb,'
'tests,transforms,doc,register). Forces --clean to false'
)

args = parser.parse_args()
Expand All @@ -111,6 +117,7 @@
- test_outputs_path: {args.test_outputs_path}
- github_base_url: {args.github_base_url}
- filter: {args.filter}
- steps: {args.steps}
""", file=sys.stderr)

register_file = Path(args.register_file)
Expand All @@ -121,7 +128,7 @@
items_dir = Path(args.items_dir)

# Clean old output
if clean and not args.filter:
if clean and not args.filter and not args.steps:
for old_file in register_file, register_jsonld_fn, register_ttl_fn:
print(f"Deleting {old_file}", file=sys.stderr)
old_file.unlink(missing_ok=True)
Expand Down Expand Up @@ -167,6 +174,8 @@
print('[WARN] Could not autodetect base_url / github_base_url', file=sys.stderr)
pass

steps = args.steps.split(',') if args.steps else None

# 1. Postprocess BBs
print(f"Running postprocess...", file=sys.stderr)
postprocess(registered_items_path=items_dir,
Expand All @@ -180,7 +189,8 @@
test_outputs_path=args.test_outputs_path,
github_base_url=github_base_url,
imported_registers=imported_registers,
bb_filter=args.filter)
bb_filter=args.filter,
steps=steps)

# 2. Uplift register.json
print(f"Running semantic uplift of {register_file}", file=sys.stderr)
Expand Down
95 changes: 51 additions & 44 deletions ogc/bblocks/postprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@ def postprocess(registered_items_path: str | Path = 'registereditems',
test_outputs_path: str | Path = 'build/tests',
github_base_url: str | None = None,
imported_registers: list[str] | None = None,
bb_filter: str | None = None) -> list[BuildingBlock]:
bb_filter: str | None = None,
steps: list[str] | None = None) -> list[BuildingBlock]:

cwd = Path().resolve()

if not isinstance(test_outputs_path, Path):
test_outputs_path = Path(test_outputs_path)
test_outputs_path.mkdir(parents=True, exist_ok=True)
if not steps or 'tests' in steps:
test_outputs_path.mkdir(parents=True, exist_ok=True)

if base_url and base_url[-1] != '/':
base_url += '/'
Expand Down Expand Up @@ -125,21 +127,22 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:
bblock.metadata['sourceFiles'] = f"./{os.path.relpath(rel_files_path, output_file_root)}/"

if not light:
print(f" > Running tests for {bblock.identifier}", file=sys.stderr)
validation_passed, test_count, json_report = validate_test_resources(
bblock,
bblocks_register=bbr,
outputs_path=test_outputs_path,
base_url=base_url)
validation_reports.append(json_report)

bblock.metadata['validationPassed'] = validation_passed
if not validation_passed:
bblock.metadata['status'] = 'invalid'
if test_count and test_outputs_base_url:
bblock.metadata['testOutputs'] = f"{test_outputs_base_url}{bblock.subdirs}/"

if not light:
if not steps or 'tests' in steps:
print(f" > Running tests for {bblock.identifier}", file=sys.stderr)
validation_passed, test_count, json_report = validate_test_resources(
bblock,
bblocks_register=bbr,
outputs_path=test_outputs_path,
base_url=base_url)
validation_reports.append(json_report)

bblock.metadata['validationPassed'] = validation_passed
if not validation_passed:
bblock.metadata['status'] = 'invalid'
if test_count and test_outputs_base_url:
bblock.metadata['testOutputs'] = f"{test_outputs_base_url}{bblock.subdirs}/"

if not light and (not steps or 'transforms' in steps):
print(f" > Running transforms for {bblock.identifier}", file=sys.stderr)
apply_transforms(bblock, outputs_path=test_outputs_path)

Expand All @@ -162,7 +165,7 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:
transform['ref'] = urljoin(bblock.metadata['sourceFiles'], transform['ref'])
bblock.metadata['transforms'].append({k: v for k, v in transform.items() if k != 'code'})

if not light:
if not light and (not steps or 'doc' in steps):
print(f" > Generating documentation for {bblock.identifier}", file=sys.stderr)
doc_generator.generate_doc(bblock)
return True
Expand Down Expand Up @@ -199,7 +202,8 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:
super_bblocks[building_block.files_path] = building_block
continue

if filter_id is None or building_block.identifier == filter_id:
if (filter_id is None or building_block.identifier == filter_id) and (not steps or 'annotate' in steps):

# Annotate schema
print(f"Annotating schema for {building_block.identifier}", file=sys.stderr)

Expand Down Expand Up @@ -229,31 +233,33 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:

child_bblocks.append(building_block)

print(f"Writing JSON-LD contexts", file=sys.stderr)
# Create JSON-lD contexts
for building_block in child_bblocks:
if filter_id is not None and building_block.identifier != filter_id:
continue
if building_block.annotated_schema.is_file():
try:
written_context = write_jsonld_context(building_block.annotated_schema)
if written_context:
print(f" - {written_context}", file=sys.stderr)
except Exception as e:
if fail_on_error:
raise e
print(f"[Error] Writing context for {building_block.identifier}: {type(e).__name__}: {e}")
if not steps or 'jsonld' in steps:
print(f"Writing JSON-LD contexts", file=sys.stderr)
# Create JSON-lD contexts
for building_block in child_bblocks:
if filter_id is not None and building_block.identifier != filter_id:
continue
if building_block.annotated_schema.is_file():
try:
written_context = write_jsonld_context(building_block.annotated_schema)
if written_context:
print(f" - {written_context}", file=sys.stderr)
except Exception as e:
if fail_on_error:
raise e
print(f"[Error] Writing context for {building_block.identifier}: {type(e).__name__}: {e}")

# Create super bblock schemas
# TODO: Do not build super bb's that have children with errors
print(f"Generating Super Building Block schemas", file=sys.stderr)
try:
for super_bblock_schema in write_superbblocks_schemas(super_bblocks, annotated_path):
print(f" - {os.path.relpath(super_bblock_schema, '.')}", file=sys.stderr)
except Exception as e:
if fail_on_error:
raise e
print(f"[Error] Writing Super BB schemas: {type(e).__name__}: {e}")
if not steps or 'superbb' in steps:
print(f"Generating Super Building Block schemas", file=sys.stderr)
try:
for super_bblock_schema in write_superbblocks_schemas(super_bblocks, annotated_path):
print(f" - {os.path.relpath(super_bblock_schema, '.')}", file=sys.stderr)
except Exception as e:
if fail_on_error:
raise e
print(f"[Error] Writing Super BB schemas: {type(e).__name__}: {e}")

output_bblocks = []
for building_block in itertools.chain(child_bblocks, super_bblocks.values()):
Expand All @@ -265,10 +271,11 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:
else:
print(f"{building_block.identifier} failed postprocessing, skipping...", file=sys.stderr)

print(f"Writing full validation report to {test_outputs_path / 'report.html'}", file=sys.stderr)
report_to_html(json_reports=validation_reports, report_fn=test_outputs_path / 'report.html')
if not steps or 'tests' in steps:
print(f"Writing full validation report to {test_outputs_path / 'report.html'}", file=sys.stderr)
report_to_html(json_reports=validation_reports, report_fn=test_outputs_path / 'report.html')

if output_file:
if output_file and (not steps or 'register' in steps):
output_register_json = {
'imports': imported_registers or [],
'bblocks': output_bblocks,
Expand Down

0 comments on commit 79d4881

Please sign in to comment.