From 33b8063835b206c2a85206392d5c9ebc00f3235b Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Thu, 27 Oct 2022 17:55:34 +0200 Subject: [PATCH 01/74] Resolution of some bugs in the sdmx2jsonld python package and configuration of poetry bumpversion --- agent.py | 6 +++--- cli/command.py | 2 +- pyproject.toml | 14 ++++++++++---- sdmx2jsonld/README.md | 8 ++++++-- sdmx2jsonld/common/config.py | 16 ++++++++++++++++ sdmx2jsonld/transform/parser.py | 5 +++-- 6 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 sdmx2jsonld/common/config.py diff --git a/agent.py b/agent.py index 7461069..a56f3fb 100644 --- a/agent.py +++ b/agent.py @@ -30,12 +30,12 @@ if args['run'] is True: file_in = args['--input'] - file_out = args['--output'] + generate_files = args['--output'] - myparser = Parser() + my_parser = Parser() try: - myparser.parsing(content=file_in, out=file_out) + my_parser.parsing(content=file_in, out=generate_files) except UnexpectedToken as e: print(e) except UnexpectedInput as e: diff --git a/cli/command.py b/cli/command.py index 0742f17..b7fd47f 100644 --- a/cli/command.py +++ b/cli/command.py @@ -50,7 +50,7 @@ from schema import Schema, And, Or, Use, SchemaError -__version__ = "0.5.0" +__version__ = "0.5.2" __author__ = "fla" diff --git a/pyproject.toml b/pyproject.toml index 59efb59..257dda7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "SDMX2JSON-LD" -version = "0.5.0" +version = "0.5.2" description = "A SDMX in RDF Turtle 1.1 format parser to generate valid JSON-LD and send to FIWARE Context Brokers using ETSI NGSI-LD." authors = ["Fernando López "] readme = "./sdmx2jsonld/README.md" @@ -18,13 +18,13 @@ repository = "https://github.com/flopezag/IoTAgent-Turtle" [tool.poetry.dependencies] python = "^3.10" -lark = "1.1.2" +lark = "1.1.3" secure = "0.3.0" docopt = "0.6.2" schema = "0.7.5" hi-dateinfer = "0.4.6" -fastapi = "0.85.0" -uvicorn = "0.18.3" +fastapi = "0.85.1" +uvicorn = "0.19.0" python-multipart = "0.0.5" loguru = "0.6.0" requests = "2.28.1" @@ -34,3 +34,9 @@ rdflib = "~6.2.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + + +[[tool.poetry_bumpversion.replacements]] +files = ["cli/command.py", "sdmx2jsonld/transform/parser.py"] +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' diff --git a/sdmx2jsonld/README.md b/sdmx2jsonld/README.md index 623c950..d77c59c 100644 --- a/sdmx2jsonld/README.md +++ b/sdmx2jsonld/README.md @@ -41,7 +41,8 @@ alt="Logo" width="280" height="160"> A SDMX in RDF Turtle 1.1 format parser to generate valid JSON-LD and send to FIWARE Context Brokers using ETSI NGSI-LD. -It is based on a [EBNF LALR(1) grammar](https://github.com/flopezag/IoTAgent-Turtle/blob/master/grammar/grammar.lark). +It is based on a +[EBNF LALR(1) grammar](https://github.com/flopezag/IoTAgent-Turtle/blob/master/sdmx2jsonld/grammar/grammar.lark). This project is part of INTERSTAT. For more information about the INTERSTAT Project, please check the url https://cef-interstat.eu. @@ -91,11 +92,14 @@ content to be sent to the FIWARE Context Broker: from sdmx2jsonld.transform.parser import Parser from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken +file_in = open("structures-accounts.ttl") +generate_files = True + # Start parsing the file my_parser = Parser() try: - my_parser.parsing(content=file_in, out=file_out) + my_parser.parsing(content=file_in, out=generate_files) except UnexpectedToken as e: print(e) except UnexpectedInput as e: diff --git a/sdmx2jsonld/common/config.py b/sdmx2jsonld/common/config.py new file mode 100644 index 0000000..84b50b5 --- /dev/null +++ b/sdmx2jsonld/common/config.py @@ -0,0 +1,16 @@ +from os.path import join, exists, dirname, abspath + +# Settings file is inside Basics directory, therefore I have to go back to the parent directory +# to have the Code Home directory +MODULEHOME = dirname(dirname(abspath(__file__))) +GRAMMARFOLDER = join(MODULEHOME, 'grammar') +GRAMMARFILE = join(GRAMMARFOLDER, 'grammar.lark') + +if not exists(GRAMMARFILE): + msg = '\nERROR: There is not Lark grammar file in the expected folder. ' \ + '\n Unable to parse the RDF Turtle file.' \ + '\n\n Please correct it if you do not want to see these messages.\n\n\n' + + print(msg) + + exit(1) diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 85a3366..2d5f3fb 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -28,16 +28,17 @@ from logging import getLogger from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken from sdmx2jsonld.common.rdf import turtle_terse +from sdmx2jsonld.common.config import GRAMMARFILE logger = getLogger(__name__) -__version__ = "0.5.0" +__version__ = "0.5.2" class Parser: def __init__(self): # Open the grammar file - with open("./sdmx2jsonld/grammar/grammar.lark") as f: + with open(GRAMMARFILE) as f: grammar = f.read() self.parser = Lark(grammar, start='start', parser='lalr') From 349936effe7ed3ac9bd056f2b5ecd484544bd6fc Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Wed, 2 Nov 2022 11:42:22 +0100 Subject: [PATCH 02/74] Configuration of tox and some corrections to the Unit Test path --- .gitignore | 2 +- examples/sep-dsd-2.ttl | 2 +- poetry.lock | 634 ++++++++++++++++++ pyproject.toml | 4 +- {test => tests}/__init__.py | 0 {test => tests}/common/__init__.py | 0 .../common/test_classprecedence.py | 2 +- {test => tests}/docker-compose.yml | 0 tox.ini | 6 + 9 files changed, 646 insertions(+), 4 deletions(-) create mode 100644 poetry.lock rename {test => tests}/__init__.py (100%) rename {test => tests}/common/__init__.py (100%) rename {test => tests}/common/test_classprecedence.py (98%) rename {test => tests}/docker-compose.yml (100%) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index bb0332d..795f69b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,7 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports +# Unit tests / coverage reports htmlcov/ .tox/ .nox/ diff --git a/examples/sep-dsd-2.ttl b/examples/sep-dsd-2.ttl index cc71d83..7228ce5 100644 --- a/examples/sep-dsd-2.ttl +++ b/examples/sep-dsd-2.ttl @@ -13,7 +13,7 @@ # Data Structure Definitions for the Interstat SEP pilot # ################################################################################## -# This DSD will be used for the DDI-CDI/NGSI-LD interoperability test +# This DSD will be used for the DDI-CDI/NGSI-LD interoperability tests # It is a simple municipalitiy x sex x age group 3 dimensional cube isc:dsd1 diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..52df914 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,634 @@ +[[package]] +name = "anyio" +version = "3.6.2" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16,<0.22)"] + +[[package]] +name = "attrs" +version = "22.1.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] + +[[package]] +name = "certifi" +version = "2022.9.24" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "charset-normalizer" +version = "2.1.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +unicode-backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "contextlib2" +version = "21.6.0" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "docopt" +version = "0.6.2" +description = "Pythonic argument parser, that will make you smile" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "exceptiongroup" +version = "1.0.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastapi" +version = "0.85.1" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" +starlette = "0.20.4" + +[package.extras] +all = ["email-validator (>=1.1.1,<2.0.0)", "itsdangerous (>=1.1.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "orjson (>=3.2.1,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] +dev = ["autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "pre-commit (>=2.17.0,<3.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer (>=0.4.1,<0.7.0)"] +test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.971)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "types-orjson (==3.6.2)", "types-ujson (==5.4.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "hi-dateinfer" +version = "0.4.6" +description = "Infers date format from examples, by using a series of pattern matching and rewriting rules to compute a 'best guess' datetime.strptime format string give a list of example date strings." +category = "main" +optional = false +python-versions = "!=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +pytz = "*" +wheel = "*" + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + +[[package]] +name = "lark" +version = "1.1.3" +description = "a modern parsing library" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +atomic-cache = ["atomicwrites"] +nearley = ["js2py"] +regex = ["regex"] + +[[package]] +name = "loguru" +version = "0.6.0" +description = "Python logging made (stupidly) simple" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.2" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pytest" +version = "7.2.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "python-multipart" +version = "0.0.5" +description = "A streaming multipart parser for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.4.0" + +[[package]] +name = "pytz" +version = "2022.6" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "rdflib" +version = "6.2.0" +description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +isodate = "*" +pyparsing = "*" +setuptools = "*" + +[package.extras] +berkeleydb = ["berkeleydb"] +dev = ["black (==22.6.0)", "flake8", "flakeheaven", "isort", "mypy", "pep8-naming", "types-setuptools"] +docs = ["myst-parser", "sphinx (<6)", "sphinx-autodoc-typehints", "sphinxcontrib-apidoc", "sphinxcontrib-kroki"] +html = ["html5lib"] +networkx = ["networkx"] +tests = ["html5lib", "pytest", "pytest-cov"] + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "schema" +version = "0.7.5" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +contextlib2 = ">=0.5.5" + +[[package]] +name = "secure" +version = "0.3.0" +description = "A lightweight package that adds security headers for Python web frameworks." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "setuptools" +version = "65.5.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "starlette" +version = "0.20.4" +description = "The little ASGI library that shines." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.12" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "uvicorn" +version = "0.19.0" +description = "The lightning-fast ASGI server." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.0)"] + +[[package]] +name = "wheel" +version = "0.37.1" +description = "A built-package format for Python" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.extras] +test = ["pytest (>=3.0.0)", "pytest-cov"] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "efb208b9ebb656a11935975f1f40e583c2c4ac285bbb8a5509f560da21396f7e" + +[metadata.files] +anyio = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] +certifi = [ + {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +contextlib2 = [ + {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, + {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, +] +docopt = [ + {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, + {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, +] +fastapi = [ + {file = "fastapi-0.85.1-py3-none-any.whl", hash = "sha256:de3166b6b1163dc22da4dc4ebdc3192fcbac7700dd1870a1afa44de636a636b5"}, + {file = "fastapi-0.85.1.tar.gz", hash = "sha256:1facd097189682a4ff11cbd01334a992e51b56be663b2bd50c2c09523624f144"}, +] +h11 = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] +hi-dateinfer = [ + {file = "hi-dateinfer-0.4.6.tar.gz", hash = "sha256:e2ded27aeb15ee6fcda0a30c2a341acbdaaa5b95c1c4db4c4680a4dbcda35c54"}, + {file = "hi_dateinfer-0.4.6-py3-none-any.whl", hash = "sha256:46d27ccc875890315eec9d71d76c2306e84388c4a88106d2f768a921debb49e8"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +isodate = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] +lark = [ + {file = "lark-1.1.3-py3-none-any.whl", hash = "sha256:45cd8b4d8b0487863f0f4cf3c305a1293db86af576d4a62cfb572e57a9b609e5"}, + {file = "lark-1.1.3.tar.gz", hash = "sha256:ca0d1aeb57f434c7276d209729e92b0e5017d177dde553134760c35bb4647d11"}, +] +loguru = [ + {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, + {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +pydantic = [ + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pytest = [ + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, +] +python-multipart = [ + {file = "python-multipart-0.0.5.tar.gz", hash = "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43"}, +] +pytz = [ + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, +] +rdflib = [ + {file = "rdflib-6.2.0-py3-none-any.whl", hash = "sha256:85c34a86dfc517a41e5f2425a41a0aceacc23983462b32e68610b9fad1383bca"}, + {file = "rdflib-6.2.0.tar.gz", hash = "sha256:62dc3c86d1712db0f55785baf8047f63731fa59b2682be03219cb89262065942"}, +] +requests = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] +schema = [ + {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, + {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, +] +secure = [ + {file = "secure-0.3.0-py3-none-any.whl", hash = "sha256:a93b720c7614809c131ca80e477263140107c6c212829d0a6e1f7bc8d859c608"}, + {file = "secure-0.3.0.tar.gz", hash = "sha256:6e30939d8f95bf3b8effb8a36ebb5ed57f265daeeae905e3aa9677ea538ab64e"}, +] +setuptools = [ + {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, + {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +sniffio = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] +starlette = [ + {file = "starlette-0.20.4-py3-none-any.whl", hash = "sha256:c0414d5a56297d37f3db96a84034d61ce29889b9eaccf65eb98a0b39441fcaa3"}, + {file = "starlette-0.20.4.tar.gz", hash = "sha256:42fcf3122f998fefce3e2c5ad7e5edbf0f02cf685d646a83a08d404726af5084"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +typing-extensions = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] +urllib3 = [ + {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, +] +uvicorn = [ + {file = "uvicorn-0.19.0-py3-none-any.whl", hash = "sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f"}, + {file = "uvicorn-0.19.0.tar.gz", hash = "sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25"}, +] +wheel = [ + {file = "wheel-0.37.1-py2.py3-none-any.whl", hash = "sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a"}, + {file = "wheel-0.37.1.tar.gz", hash = "sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4"}, +] +win32-setctime = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] diff --git a/pyproject.toml b/pyproject.toml index 257dda7..95556a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ repository = "https://github.com/flopezag/IoTAgent-Turtle" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.7" lark = "1.1.3" secure = "0.3.0" docopt = "0.6.2" @@ -30,6 +30,8 @@ loguru = "0.6.0" requests = "2.28.1" rdflib = "~6.2.0" +[tool.poetry.dev-dependencies] +pytest = "*" [build-system] requires = ["poetry-core"] diff --git a/test/__init__.py b/tests/__init__.py similarity index 100% rename from test/__init__.py rename to tests/__init__.py diff --git a/test/common/__init__.py b/tests/common/__init__.py similarity index 100% rename from test/common/__init__.py rename to tests/common/__init__.py diff --git a/test/common/test_classprecedence.py b/tests/common/test_classprecedence.py similarity index 98% rename from test/common/test_classprecedence.py rename to tests/common/test_classprecedence.py index d4b9c8a..974b3cf 100644 --- a/test/common/test_classprecedence.py +++ b/tests/common/test_classprecedence.py @@ -20,7 +20,7 @@ # under the License. ## from unittest import TestCase -from common.classprecedence import Precedence, ClassesPrecedencePropertyError, ClassesPrecedenceClassError +from sdmx2jsonld.common.classprecedence import Precedence, ClassesPrecedencePropertyError, ClassesPrecedenceClassError class Test(TestCase): diff --git a/test/docker-compose.yml b/tests/docker-compose.yml similarity index 100% rename from test/docker-compose.yml rename to tests/docker-compose.yml diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..457561e --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py37, py38, py39, py310, py311 + +[testenv] +commands = + pytest tests/ \ No newline at end of file From 5f17f217e993de935d85123574277e8740207019 Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Thu, 10 Nov 2022 10:12:33 +0100 Subject: [PATCH 03/74] Adding Dataset->CatalogueDCAT-AP and Observation and controlling some issues reading files --- examples/observation.ttl | 47 ++++++++++ examples/sep-dsd-2.ttl | 2 +- sdmx2jsonld/common/datatypeconversion.py | 27 ++++-- sdmx2jsonld/transform/entitytype.py | 4 + sdmx2jsonld/transform/parser.py | 8 +- tests/common/test_datatypeconversion.py | 113 +++++++++++++++++++++++ 6 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 examples/observation.ttl create mode 100644 tests/common/test_datatypeconversion.py diff --git a/examples/observation.ttl b/examples/observation.ttl new file mode 100644 index 0000000..f7cc9b8 --- /dev/null +++ b/examples/observation.ttl @@ -0,0 +1,47 @@ +@prefix rdf: . +@prefix dc: . +@prefix dcterms: . +@prefix qb: . +@prefix rdfs: . +@prefix owl: . +@prefix skos: . +@prefix xsd: . +@prefix sdmx: . +@prefix sdmx-concept: . +@prefix sdmx-dimension: . +@prefix sdmx-attribute: . +@prefix sdmx-measure: . +@prefix sdmx-metadata: . +@prefix sdmx-code: . +@prefix sdmx-subject: . + + a qb:DataSet ; +dcterms:issued "2022-04-01T08:00:00.000"^^xsd:dateTime ; +dcterms:publisher ; +dcterms:title "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr ; +qb:structure ; +rdfs:label "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr ; +sdmx-attribute:title "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr . + + a qb:Observation; + ; + "W2" ; + "S1" ; + "S1" ; + "B" ; + "B1G" ; + "_Z" ; + "A" ; + "_Z" ; + "XDC" ; + "V" ; + "N" ; +qb:dataSet ; +sdmx-attribute:confStatus sdmx-code:confStatus-F ; +sdmx-attribute:decimals sdmx-code:decimals-1 ; +sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; +sdmx-attribute:unitMult sdmx-code:unitMult-6 ; +sdmx-dimension:freq sdmx-code:freq-A ; +sdmx-dimension:refArea "BE" ; +sdmx-dimension:timePeriod "2012" ; +sdmx-measure:obsValue "3016.9"^^xsd:float . diff --git a/examples/sep-dsd-2.ttl b/examples/sep-dsd-2.ttl index 7228ce5..fd18a89 100644 --- a/examples/sep-dsd-2.ttl +++ b/examples/sep-dsd-2.ttl @@ -14,7 +14,7 @@ ################################################################################## # This DSD will be used for the DDI-CDI/NGSI-LD interoperability tests -# It is a simple municipalitiy x sex x age group 3 dimensional cube +# It is a simple municipality x sex x age group 3 dimensional cube isc:dsd1 a qb:DataStructureDefinition ; diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index 6b6f8a0..f383b16 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -29,8 +29,10 @@ def __init__(self): self.types = { 'xsd:dateTime': 'stodt', 'xsd:int': 'stoi', - 'xsd:boolean': 'stob' + 'xsd:boolean': 'stob', + 'xsd:float': 'stof' } + self.regex_12hour = compile(r"(^.*T%)(I)(.*)$") self.regex_microseconds = compile(r"^(.*T%.*:%S\.)(%H)*$") self.regex_microseconds2 = compile(r"^(.*T%.*:%S\.)(%y)*$") @@ -52,7 +54,6 @@ def correct_datatype_format(self, format_dt: str, hour24: bool = True): def convert(self, data, datatype): def stodt(value): - # print(f'toDateTime function, arguments {value}') if isinstance(value, str): result = infer([value]) elif isinstance(value, list): @@ -61,10 +62,8 @@ def stodt(value): raise Exception(f'Invalid format received: {type(value)}') result = self.correct_datatype_format(result) - - # print(f'format {result}') result = datetime.strptime(value, result).replace(tzinfo=timezone.utc).isoformat() - # print(f'result {result}') + return result def stoi(value): @@ -80,14 +79,25 @@ def stoi(value): return int(result) + def stof(value): + """ + Converts 'something' to float. Raises exception for invalid formats + """ + if isinstance(value, str): + result = value.replace('"', '') + elif isinstance(value, float): + result = value + else: + raise Exception(f'Invalid format received: {type(value)}') + + return float(result) + def stob(value): """ Converts 'something' to boolean. Raises exception for invalid formats Possible True values: 1, True, "1", "TRue", "yes", "y", "t" Possible False values: 0, False, None, [], {}, "", "0", "faLse", "no", "n", "f", 0.0, ... """ - print(f'toBool function, arguments {value}') - if str(value).lower() in ("yes", "y", "true", "t", "1"): return True @@ -158,3 +168,6 @@ def stob(value): print(dataConversionType.convert(data6[0], data6[2])) print(dataConversionType.convert(data7[0], data7[2])) + + data101 = ['"3016.9"', Token('FORMATCONNECTOR', '^^'), 'xsd:float'] + print(dataConversionType.convert(data101[0], data101[2])) diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 16c861c..5e23628 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -37,6 +37,8 @@ class EntityType: def __init__(self): self.entities = { + 'qb:DataSet': 'Catalogue', + 'qb:Observation': 'Observation', 'qb:DataStructureDefinition': 'Dataset', 'qb:ComponentSpecification': 'Component', 'qb:AttributeProperty': 'Attribute', @@ -142,6 +144,8 @@ def create_data(self, type, data, title): if type == 'Component': self.dataset.add_components(context=self.context, component=data) + elif type == 'Catalogue': + print(title) elif type == 'Dataset': identifier = parser.obtain_id(title) self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 2d5f3fb..7f7ebd0 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -62,11 +62,13 @@ def parsing(self, content: TextIOBase, out: bool = False): def parsing_file(self, content: TextIOWrapper, out: bool): transform = TreeToJson() - content = content.read() - content = turtle_terse(rdf_content=content) + with content as f: + data = f.read() + + data = turtle_terse(rdf_content=data) try: - tree = self.parser.parse(content) + tree = self.parser.parse(data) except UnexpectedToken as err: raise err except UnexpectedInput as err: diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py new file mode 100644 index 0000000..960d029 --- /dev/null +++ b/tests/common/test_datatypeconversion.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from unittest import TestCase +from sdmx2jsonld.common.datatypeconversion import DataTypeConversion + + +class Test(TestCase): + def setUp(self): + self.conversion = DataTypeConversion() + + def test_datetime_string_conversion_1(self): + """ + Check if we can get a correct datetime value from a string, case 1 + """ + obtained = self.conversion.convert('"2022-01-15T08:00:00.000"', 'xsd:dateTime') + expected = '2022-01-15T08:00:00+00:00' + assert obtained == expected, f"\n\nDateTime was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_int_string_conversion(self): + """ + Check if we can get a correct integer from a string + """ + obtained = self.conversion.convert('"2"', 'xsd:int') + expected = 2 + assert obtained == expected, f"\n\nInteger was not the expected," \ + f"\n got : {obtained} {type(obtained)}" \ + f"\n expected: {expected} {type(expected)}" + + def test_int_integer_conversion(self): + """ + Check if we can get a correct integer from an integer + """ + obtained = self.conversion.convert('2', 'xsd:int') + expected = 2 + assert obtained == expected, f"\n\nInteger was not the expected," \ + f"\n got : {obtained} {type(obtained)}" \ + f"\n expected: {expected} {type(expected)}" + + # # print(dataConversionType.convert(data23[0], data23[2]) + 10) + # + + def test_boolean_conversion(self): + """ + Check if we can convert a boolean string into its proper value + """ + obtained = self.conversion.convert('"true"', 'xsd:boolean') + expected = True + assert obtained == expected, f"\n\nBoolean was not the expected," \ + f"\n got : {obtained} {type(obtained)}" \ + f"\n expected: {expected} {type(expected)}" + + def test_fake_conversion(self): + """ + Check is a fake value data launch an exception + """ + with self.assertRaises(Exception) as error: + _ = self.conversion.convert('"fake"', 'otraCosa') + + self.assertEqual(str(error.exception), + "Datatype not defined: otraCosa") + + # # Convert datetime generated into UTC format: 2021-12-21T16:18:55Z or 2021-12-21T16:18:55+00:00, ISO8601 + # + # data5 = ['"2022-01-10T09:00:00.000"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] + # print(dataConversionType.convert(data5[0], data5[2])) + # + # data6 = ['"2021-07-01T11:50:37.3"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] + # print(dataConversionType.convert(data6[0], data6[2])) + # + # data7 = ['"2021-09-28T15:31:24.05"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] + # print(dataConversionType.convert(data7[0], data7[2])) + # + + def test_float_float_conversion(self): + """ + Check if we can get a correct integer from an string + """ + obtained = self.conversion.convert('2345.2', 'xsd:float') + expected = 2345.2 + assert obtained == expected, f"\n\nInteger was not the expected," \ + f"\n got : {obtained} {type(obtained)}" \ + f"\n expected: {expected} {type(expected)}" + + def test_float_string_conversion(self): + """ + Check if we can get a correct integer from an integer + """ + obtained = self.conversion.convert('"3016.9"', 'xsd:float') + expected = 3016.9 + assert obtained == expected, f"\n\nFloat was not the expected," \ + f"\n got : {obtained} {type(obtained)}" \ + f"\n expected: {expected} {type(expected)}" From 91ba51ab530f3d190748d5c24795cf65e3f86fd5 Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Thu, 10 Nov 2022 10:52:48 +0100 Subject: [PATCH 04/74] Correct save Dataset when there is no id, no content and generate the first version of CatalogueDCAT-AP for observation content --- sdmx2jsonld/common/listmanagement.py | 2 +- sdmx2jsonld/transform/catalogue.py | 86 ++++++++++++++++++++++++++-- sdmx2jsonld/transform/entitytype.py | 8 ++- sdmx2jsonld/transform/transformer.py | 3 +- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index eeae59f..131f200 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -55,7 +55,7 @@ def flatten_value(y): return y.replace('"', '') -def get_rest_data(data, not_allowed_keys, further_process_keys): +def get_rest_data(data, not_allowed_keys=[], further_process_keys=[]): aux = {data[i]: flatten_value(data[i + 1]) for i in range(0, len(data), 2)} # We need to get the list of keys from the dict diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7502ee0..0046d80 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -22,6 +22,7 @@ from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.common.listmanagement import get_rest_data import random logger = getLogger() @@ -34,12 +35,12 @@ def __init__(self): self.data = { "id": str(), "type": "CatalogueDCAT-AP", - "dataset": { + "qb:dataset": { "type": "object", "value": str() }, - "language": { + "dct:language": { "type": "Property", "value": list() }, @@ -54,17 +55,17 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "description": { + "dct:description": { "type": "Property", "value": dict() }, - "publisher": { + "dct:publisher": { "type": "Property", "value": str() }, - "title": { + "dct:title": { "type": "Array", "value": list() }, @@ -77,6 +78,7 @@ def __init__(self): } self.concept_id = str() + self.keys = {k: k for k in self.data.keys()} def add_dataset(self, dataset_id): self.concept_id = dataset_id @@ -91,8 +93,82 @@ def add_dataset(self, dataset_id): # Add dataset id self.data['dataset']['value'] = dataset_id + def add_data(self, title, dataset_id, data): + # We need to complete the data corresponding to the Catalogue: rdfs:label + self.__complete_label__(title=title, data=data) + + # Add the title + key = self.keys['dct:title'] + self.data[key] = title + + # Add the id + self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + dataset_id + + # Get the rest of the data + data = get_rest_data(data=data) + + # add the new data to the dataset structure + self.patch_data(data, False) + + def patch_data(self, data, language_map): + if language_map: + self.__complete_label__(title="Not specified", data=data) + else: + # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide + # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. + [self.data.update({k: v}) for k, v in data.items()] + + def __complete_label__(self, title, data): + try: + key = self.get_key(requested_key='rdfs:label') + position = data.index(key) + 1 + description = data[position] + + descriptions = [x[0].replace("\"", "") for x in description] + + languages = list() + try: + languages = [x[1].replace("@", "").lower() for x in description] + except IndexError: + logger.warning(f'The Catalogue {title} has a ' + f'rdfs:label without language tag: {description}') + + aux = len(description) + if aux != 1: + logger.error(f"Catalogue: there is more than 1 description ({aux}), values: {description}") + else: + # There is no language tag, we use by default 'en' + languages = ['en'] + logger.warning('Catalogue: selecting default language "en"') + + ############################################################################### + # TODO: New ETSI CIM NGSI-LD specification 1.4.2 + # Pending to implement in the Context Broker + ############################################################################### + # for i in range(0, len(languages)): + # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] + ############################################################################### + for i in range(0, len(languages)): + key = self.keys['dct:description'] + self.data[key]['value'][languages[i]] = descriptions[i] + + # Complete the information of the language with the previous information + key = self.keys['dct:language'] + self.data[key]['value'] = languages + except ValueError: + logger.info(f'Dataset without rdfs:label detail: {title}') + def get(self): return self.data def get_id(self): return self.data['id'] + + def get_key(self, requested_key): + try: + key = self.keys[requested_key] + return key + except KeyError: + # The key did not exist therefore we add to the list with this value + self.keys[requested_key] = requested_key + return requested_key diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 5e23628..161472c 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -145,7 +145,13 @@ def create_data(self, type, data, title): if type == 'Component': self.dataset.add_components(context=self.context, component=data) elif type == 'Catalogue': - print(title) + identifier = parser.obtain_id(title) + self.catalogue.add_data(title=title, dataset_id=identifier, data=data) + + print(identifier) + elif type == 'Observation': + identifier = parser.obtain_id(title) + print(identifier) elif type == 'Dataset': identifier = parser.obtain_id(title) self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index b20e70d..a650944 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -133,7 +133,8 @@ def get_conceptLists(self): def save(self): self.entity_type.save('catalogue') - self.entity_type.save('dataset') + if self.entity_type.dataset.data['id'] != '': + self.entity_type.save('dataset') dimensions = self.entity_type.get_dimensions() [dimension.save() for dimension in dimensions] From 47c0821acf52e6d5fdc255c985b6b76aa520f64e Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Fri, 9 Dec 2022 14:44:19 +0100 Subject: [PATCH 05/74] Added some tests and corrected some code from tests --- .gitignore | 1 + sdmx2jsonld/common/classprecedence.py | 2 +- sdmx2jsonld/common/commonclass.py | 5 +- sdmx2jsonld/common/datatypeconversion.py | 19 +- sdmx2jsonld/common/tzinfos.py | 229 +++++++++++++++++++++++ tests/common/test_commonclass.py | 45 +++++ tests/common/test_datatypeconversion.py | 76 ++++++++ tests/kex/e1.ttl | 8 + 8 files changed, 377 insertions(+), 8 deletions(-) create mode 100644 sdmx2jsonld/common/tzinfos.py create mode 100644 tests/common/test_commonclass.py create mode 100644 tests/common/test_datatypeconversion.py create mode 100644 tests/kex/e1.ttl diff --git a/.gitignore b/.gitignore index 795f69b..eacca3c 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ dmypy.json *.jsonld output output/*.jsonld +tests/test1.py diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py index bf642c9..edbfbe2 100644 --- a/sdmx2jsonld/common/classprecedence.py +++ b/sdmx2jsonld/common/classprecedence.py @@ -71,7 +71,7 @@ def precedence(self, data: list) -> str: if result is True: raise ClassesPrecedenceClassError(data) - # In other chase, we return the max value of the list + # In other case, we return the max value of the list aux = max(classes_values) aux = data[classes_values.index(aux)] diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index 3aea878..f58b66d 100644 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -53,6 +53,9 @@ def add_context(self, context, context_mapping): self.data = new_data + def get(self): + return self.data + def save(self): data = self.get() @@ -61,7 +64,7 @@ def save(self): # We need to check that the output folder exist if exists('./output') is False: - # We need to create the folder because it does not exits + # We need to create the folder because it does not exist mkdir('./output') filename = './output/' + '_'.join(aux[length_aux - 2:]) + '.jsonld' diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index 6b6f8a0..0128405 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -22,12 +22,14 @@ from hidateinfer import infer from datetime import datetime, timezone from re import compile, sub - +from dateutil import parser +import pytz +from sdmx2jsonld.common.tzinfos import whois_timezone_info class DataTypeConversion: def __init__(self): self.types = { - 'xsd:dateTime': 'stodt', + 'xsd:dateTime': 'stoutc', 'xsd:int': 'stoi', 'xsd:boolean': 'stob' } @@ -42,15 +44,20 @@ def correct_datatype_format(self, format_dt: str, hour24: bool = True): if hour24: format_dt = sub(self.regex_12hour, r"\1H\3", format_dt) - format_dt = sub(self.regex_microseconds, r"\1%f", format_dt) + format_dt = sub(self.regex_microseconds, r"\1%f", format_dt) format_dt = sub(self.regex_microseconds2, r"\1%f", format_dt) - format_dt = sub(self.regex_false_date, r"%Y-%m-%d\1%f", format_dt) + format_dt = sub(self.regex_false_date, r"%Y-%m-%d\1%f", format_dt) format_dt = sub(self.regex_false_date2, r"%Y-%m-%d\1%f", format_dt) return format_dt def convert(self, data, datatype): + def stoutc(value): + dt = parser.parse(value, tzinfos=whois_timezone_info) + dt = dt.astimezone(pytz.UTC) + return dt.replace(tzinfo=timezone.utc).isoformat() + def stodt(value): # print(f'toDateTime function, arguments {value}') if isinstance(value, str): @@ -60,7 +67,7 @@ def stodt(value): else: raise Exception(f'Invalid format received: {type(value)}') - result = self.correct_datatype_format(result) + # result = self.correct_datatype_format(result) # print(f'format {result}') result = datetime.strptime(value, result).replace(tzinfo=timezone.utc).isoformat() @@ -98,7 +105,7 @@ def stob(value): raise Exception(f'Invalid value for boolean conversion: {str(value)}') try: - function = self.types[datatype] + '(value=' + data + ')' + function = self.types[datatype] + f'(value="{data}")' return eval(function) except KeyError: # logger.error(f'Datatype not defined: {datatype}') diff --git a/sdmx2jsonld/common/tzinfos.py b/sdmx2jsonld/common/tzinfos.py new file mode 100644 index 0000000..e78012b --- /dev/null +++ b/sdmx2jsonld/common/tzinfos.py @@ -0,0 +1,229 @@ +whois_timezone_info = { + "A": 1 * 3600, + "ACDT": 10.5 * 3600, + "ACST": 9.5 * 3600, + "ACT": -5 * 3600, + "ACWST": 8.75 * 3600, + "ADT": 4 * 3600, + "AEDT": 11 * 3600, + "AEST": 10 * 3600, + "AET": 10 * 3600, + "AFT": 4.5 * 3600, + "AKDT": -8 * 3600, + "AKST": -9 * 3600, + "ALMT": 6 * 3600, + "AMST": -3 * 3600, + "AMT": -4 * 3600, + "ANAST": 12 * 3600, + "ANAT": 12 * 3600, + "AQTT": 5 * 3600, + "ART": -3 * 3600, + "AST": 3 * 3600, + "AT": -4 * 3600, + "AWDT": 9 * 3600, + "AWST": 8 * 3600, + "AZOST": 0 * 3600, + "AZOT": -1 * 3600, + "AZST": 5 * 3600, + "AZT": 4 * 3600, + "AoE": -12 * 3600, + "B": 2 * 3600, + "BNT": 8 * 3600, + "BOT": -4 * 3600, + "BRST": -2 * 3600, + "BRT": -3 * 3600, + "BST": 6 * 3600, + "BTT": 6 * 3600, + "C": 3 * 3600, + "CAST": 8 * 3600, + "CAT": 2 * 3600, + "CCT": 6.5 * 3600, + "CDT": -5 * 3600, + "CEST": 2 * 3600, + "CET": 1 * 3600, + "CHADT": 13.75 * 3600, + "CHAST": 12.75 * 3600, + "CHOST": 9 * 3600, + "CHOT": 8 * 3600, + "CHUT": 10 * 3600, + "CIDST": -4 * 3600, + "CIST": -5 * 3600, + "CKT": -10 * 3600, + "CLST": -3 * 3600, + "CLT": -4 * 3600, + "COT": -5 * 3600, + "CST": -6 * 3600, + "CT": -6 * 3600, + "CVT": -1 * 3600, + "CXT": 7 * 3600, + "ChST": 10 * 3600, + "D": 4 * 3600, + "DAVT": 7 * 3600, + "DDUT": 10 * 3600, + "E": 5 * 3600, + "EASST": -5 * 3600, + "EAST": -6 * 3600, + "EAT": 3 * 3600, + "ECT": -5 * 3600, + "EDT": -4 * 3600, + "EEST": 3 * 3600, + "EET": 2 * 3600, + "EGST": 0 * 3600, + "EGT": -1 * 3600, + "EST": -5 * 3600, + "ET": -5 * 3600, + "F": 6 * 3600, + "FET": 3 * 3600, + "FJST": 13 * 3600, + "FJT": 12 * 3600, + "FKST": -3 * 3600, + "FKT": -4 * 3600, + "FNT": -2 * 3600, + "G": 7 * 3600, + "GALT": -6 * 3600, + "GAMT": -9 * 3600, + "GET": 4 * 3600, + "GFT": -3 * 3600, + "GILT": 12 * 3600, + "GMT": 0 * 3600, + "GST": 4 * 3600, + "GYT": -4 * 3600, + "H": 8 * 3600, + "HDT": -9 * 3600, + "HKT": 8 * 3600, + "HOVST": 8 * 3600, + "HOVT": 7 * 3600, + "HST": -10 * 3600, + "I": 9 * 3600, + "ICT": 7 * 3600, + "IDT": 3 * 3600, + "IOT": 6 * 3600, + "IRDT": 4.5 * 3600, + "IRKST": 9 * 3600, + "IRKT": 8 * 3600, + "IRST": 3.5 * 3600, + "IST": 5.5 * 3600, + "JST": 9 * 3600, + "K": 10 * 3600, + "KGT": 6 * 3600, + "KOST": 11 * 3600, + "KRAST": 8 * 3600, + "KRAT": 7 * 3600, + "KST": 9 * 3600, + "KUYT": 4 * 3600, + "L": 11 * 3600, + "LHDT": 11 * 3600, + "LHST": 10.5 * 3600, + "LINT": 14 * 3600, + "M": 12 * 3600, + "MAGST": 12 * 3600, + "MAGT": 11 * 3600, + "MART": 9.5 * 3600, + "MAWT": 5 * 3600, + "MDT": -6 * 3600, + "MHT": 12 * 3600, + "MMT": 6.5 * 3600, + "MSD": 4 * 3600, + "MSK": 3 * 3600, + "MST": -7 * 3600, + "MT": -7 * 3600, + "MUT": 4 * 3600, + "MVT": 5 * 3600, + "MYT": 8 * 3600, + "N": -1 * 3600, + "NCT": 11 * 3600, + "NDT": 2.5 * 3600, + "NFT": 11 * 3600, + "NOVST": 7 * 3600, + "NOVT": 7 * 3600, + "NPT": 5.5 * 3600, + "NRT": 12 * 3600, + "NST": 3.5 * 3600, + "NUT": -11 * 3600, + "NZDT": 13 * 3600, + "NZST": 12 * 3600, + "O": -2 * 3600, + "OMSST": 7 * 3600, + "OMST": 6 * 3600, + "ORAT": 5 * 3600, + "P": -3 * 3600, + "PDT": -7 * 3600, + "PET": -5 * 3600, + "PETST": 12 * 3600, + "PETT": 12 * 3600, + "PGT": 10 * 3600, + "PHOT": 13 * 3600, + "PHT": 8 * 3600, + "PKT": 5 * 3600, + "PMDT": -2 * 3600, + "PMST": -3 * 3600, + "PONT": 11 * 3600, + "PST": -8 * 3600, + "PT": -8 * 3600, + "PWT": 9 * 3600, + "PYST": -3 * 3600, + "PYT": -4 * 3600, + "Q": -4 * 3600, + "QYZT": 6 * 3600, + "R": -5 * 3600, + "RET": 4 * 3600, + "ROTT": -3 * 3600, + "S": -6 * 3600, + "SAKT": 11 * 3600, + "SAMT": 4 * 3600, + "SAST": 2 * 3600, + "SBT": 11 * 3600, + "SCT": 4 * 3600, + "SGT": 8 * 3600, + "SRET": 11 * 3600, + "SRT": -3 * 3600, + "SST": -11 * 3600, + "SYOT": 3 * 3600, + "T": -7 * 3600, + "TAHT": -10 * 3600, + "TFT": 5 * 3600, + "TJT": 5 * 3600, + "TKT": 13 * 3600, + "TLT": 9 * 3600, + "TMT": 5 * 3600, + "TOST": 14 * 3600, + "TOT": 13 * 3600, + "TRT": 3 * 3600, + "TVT": 12 * 3600, + "U": -8 * 3600, + "ULAST": 9 * 3600, + "ULAT": 8 * 3600, + "UTC": 0 * 3600, + "UYST": -2 * 3600, + "UYT": -3 * 3600, + "UZT": 5 * 3600, + "V": -9 * 3600, + "VET": -4 * 3600, + "VLAST": 11 * 3600, + "VLAT": 10 * 3600, + "VOST": 6 * 3600, + "VUT": 11 * 3600, + "W": -10 * 3600, + "WAKT": 12 * 3600, + "WARST": -3 * 3600, + "WAST": 2 * 3600, + "WAT": 1 * 3600, + "WEST": 1 * 3600, + "WET": 0 * 3600, + "WFT": 12 * 3600, + "WGST": -2 * 3600, + "WGT": -3 * 3600, + "WIB": 7 * 3600, + "WIT": 9 * 3600, + "WITA": 8 * 3600, + "WST": 14 * 3600, + "WT": 0 * 3600, + "X": -11 * 3600, + "Y": -12 * 3600, + "YAKST": 10 * 3600, + "YAKT": 9 * 3600, + "YAPT": 10 * 3600, + "YEKST": 6 * 3600, + "YEKT": 5 * 3600, + "Z": 0 * 3600, +} diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py new file mode 100644 index 0000000..558aae7 --- /dev/null +++ b/tests/common/test_commonclass.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +from unittest import TestCase +from sdmx2jsonld.common.commonclass import CommonClass +import os + +class TestCommonClass(TestCase): + def setUp(self) -> None: + pass + + def test_instance_class(self): + cclass = CommonClass("test.common.entity") + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id") + assert (urnid == "urn:ngsi-ld:test.common.entity:entity_id") + # urnid = cclass.generate_id("") + # print(urnid) + + def test_save(self): + cclass = CommonClass("test.common.entity") + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id") + + os.makedirs("/tmp/commonclass", exist_ok=True) + os.chdir("/tmp/commonclass") + cclass.save() + diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py new file mode 100644 index 0000000..a2cee8f --- /dev/null +++ b/tests/common/test_datatypeconversion.py @@ -0,0 +1,76 @@ +from unittest import TestCase +from sdmx2jsonld.common.datatypeconversion import DataTypeConversion +import sys + + + +class TestDataTypeConversion(TestCase): + def setUp(self) -> None: + pass + + def test_string_to_integer(self): + dtc = DataTypeConversion() + token_type = 'xsd:int' + # token = Token('FORMATCONNECTOR', "432112") + + i: int = dtc.convert("432112", token_type) + assert (i == 432112) + + i = dtc.convert("-100", token_type) + assert (i == -100) + + i = dtc.convert("19223372036854775808", token_type) + assert (i == 19223372036854775808) + assert type(i) is int + + try: + dtc.convert("invalid value", token_type) + assert (False) + except Exception: + assert (True) + + def test_string_to_bool(self): + dtc = DataTypeConversion() + token_type = 'xsd:boolean' + + values = ("True", "true", "y", "yes", "T", "1", 1, True) + for value in values: + assert (dtc.convert(value, token_type)) + + values = ("True", "true", "y", "yes", "T", "1", 1, True) + for value in values: + assert (dtc.convert(value, token_type)) + + values = ("fAlsE", "False", "N", "No", "F", "0", "0.0", "", "None", None, [], {}, 0, 0.0) + for value in values: + assert (not dtc.convert(value, token_type)) + + invalid_values = (5, 4.2, "invalid value", "nil") + for value in invalid_values: + try: + dtc.convert(value, token_type) + assert (False) + except Exception: + assert (True) + + def test_string_to_dates(self): + dtc = DataTypeConversion() + token_type = 'xsd:dateTime' + + dates = ("2022-01-15T08:00:00.000+00:00", "2022-01-10T09:00:00.000", + "2021-07-01T11:50:37.3", "2021-09-28T15:31:24.05", + "Mon Jan 13 09:52:52 MST 2014", "Thu Jun 02 11:56:53 CDT 2011", + "2022-12-12T10:00:00", "2022-05-11T10:00:00", + "Tue Dec 13 11:00:00 K 2022") + expected = ("2022-01-15T08:00:00+00:00", "2022-01-10T08:00:00+00:00", + "2021-07-01T09:50:37.300000+00:00", "2021-09-28T13:31:24.050000+00:00", + "2014-01-13T16:52:52+00:00", "2011-06-02T16:56:53+00:00", + "2022-12-12T09:00:00+00:00", "2022-05-11T08:00:00+00:00", + "2022-12-13T01:00:00+00:00") + + d = zip(dates, expected) + + for test_date, expected_date in d: + # print(test_date) + # print(dtc.convert(test_date, token_type)) + assert (expected_date == dtc.convert(test_date, token_type)) \ No newline at end of file diff --git a/tests/kex/e1.ttl b/tests/kex/e1.ttl new file mode 100644 index 0000000..7a489a3 --- /dev/null +++ b/tests/kex/e1.ttl @@ -0,0 +1,8 @@ +@prefix rdf: . +@prefix contact: . + + + rdf:type contact:Person; + contact:fullName "Eric Miller"; + contact:mailbox ; + contact:personalTitle "Dr.". From dde437c671a359a8aaac1eb970949634bd62d88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 6 Jan 2023 12:48:04 +0100 Subject: [PATCH 06/74] LICENSE --- LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 LICENSE diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 From 3c25ce54b0b0e93f4dfb5f587a52bf6364500656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 6 Jan 2023 13:44:56 +0100 Subject: [PATCH 07/74] system changed --- .dockerignore | 0 .github/dependabot.yml | 0 .github/workflows/devskim-analysis.yml | 0 .gitignore | 0 README.md | 0 agent.py | 0 api/__init__.py | 0 api/custom_logging.py | 0 api/server.py | 0 cli/__init__.py | 0 cli/command.py | 0 common/__init__.py | 0 common/config.json | 0 doc/SDMX to JSON-LD.drawio | 0 doc/api.yaml | 0 docker/Dockerfile | 0 docker/config.json | 0 docker/docker-compose.yml | 0 examples/observation.ttl | 0 examples/sep-dsd-2.ttl | 0 examples/structures-accounts.ttl | 0 examples/structures-tourism.ttl | 0 images/logo.png | Bin poetry.lock | 0 pyproject.toml | 0 requirements.txt | 6 +++--- sdmx2jsonld/README.md | 0 sdmx2jsonld/__init__.py | 0 sdmx2jsonld/common/classprecedence.py | 0 sdmx2jsonld/common/commonclass.py | 0 sdmx2jsonld/common/config.py | 0 sdmx2jsonld/common/datatypeconversion.py | 0 sdmx2jsonld/common/listmanagement.py | 0 sdmx2jsonld/common/rdf.py | 0 sdmx2jsonld/common/regparser.py | 0 sdmx2jsonld/exceptions/__init__.py | 0 sdmx2jsonld/grammar/grammar.lark | 0 sdmx2jsonld/transform/__init__.py | 0 sdmx2jsonld/transform/attribute.py | 0 sdmx2jsonld/transform/catalogue.py | 2 +- sdmx2jsonld/transform/concept.py | 0 sdmx2jsonld/transform/conceptschema.py | 0 sdmx2jsonld/transform/context.py | 0 sdmx2jsonld/transform/dataset.py | 0 sdmx2jsonld/transform/dimension.py | 0 sdmx2jsonld/transform/entitytype.py | 0 sdmx2jsonld/transform/parser.py | 0 sdmx2jsonld/transform/property.py | 0 sdmx2jsonld/transform/transformer.py | 0 tests/__init__.py | 0 tests/common/__init__.py | 0 tests/common/test_classprecedence.py | 0 tests/common/test_datatypeconversion.py | 0 tests/docker-compose.yml | 0 tox.ini | 0 55 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 .dockerignore mode change 100644 => 100755 .github/dependabot.yml mode change 100644 => 100755 .github/workflows/devskim-analysis.yml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 README.md mode change 100644 => 100755 agent.py mode change 100644 => 100755 api/__init__.py mode change 100644 => 100755 api/custom_logging.py mode change 100644 => 100755 api/server.py mode change 100644 => 100755 cli/__init__.py mode change 100644 => 100755 cli/command.py mode change 100644 => 100755 common/__init__.py mode change 100644 => 100755 common/config.json mode change 100644 => 100755 doc/SDMX to JSON-LD.drawio mode change 100644 => 100755 doc/api.yaml mode change 100644 => 100755 docker/Dockerfile mode change 100644 => 100755 docker/config.json mode change 100644 => 100755 docker/docker-compose.yml mode change 100644 => 100755 examples/observation.ttl mode change 100644 => 100755 examples/sep-dsd-2.ttl mode change 100644 => 100755 examples/structures-accounts.ttl mode change 100644 => 100755 examples/structures-tourism.ttl mode change 100644 => 100755 images/logo.png mode change 100644 => 100755 poetry.lock mode change 100644 => 100755 pyproject.toml mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 sdmx2jsonld/README.md mode change 100644 => 100755 sdmx2jsonld/__init__.py mode change 100644 => 100755 sdmx2jsonld/common/classprecedence.py mode change 100644 => 100755 sdmx2jsonld/common/commonclass.py mode change 100644 => 100755 sdmx2jsonld/common/config.py mode change 100644 => 100755 sdmx2jsonld/common/datatypeconversion.py mode change 100644 => 100755 sdmx2jsonld/common/listmanagement.py mode change 100644 => 100755 sdmx2jsonld/common/rdf.py mode change 100644 => 100755 sdmx2jsonld/common/regparser.py mode change 100644 => 100755 sdmx2jsonld/exceptions/__init__.py mode change 100644 => 100755 sdmx2jsonld/grammar/grammar.lark mode change 100644 => 100755 sdmx2jsonld/transform/__init__.py mode change 100644 => 100755 sdmx2jsonld/transform/attribute.py mode change 100644 => 100755 sdmx2jsonld/transform/catalogue.py mode change 100644 => 100755 sdmx2jsonld/transform/concept.py mode change 100644 => 100755 sdmx2jsonld/transform/conceptschema.py mode change 100644 => 100755 sdmx2jsonld/transform/context.py mode change 100644 => 100755 sdmx2jsonld/transform/dataset.py mode change 100644 => 100755 sdmx2jsonld/transform/dimension.py mode change 100644 => 100755 sdmx2jsonld/transform/entitytype.py mode change 100644 => 100755 sdmx2jsonld/transform/parser.py mode change 100644 => 100755 sdmx2jsonld/transform/property.py mode change 100644 => 100755 sdmx2jsonld/transform/transformer.py mode change 100644 => 100755 tests/__init__.py mode change 100644 => 100755 tests/common/__init__.py mode change 100644 => 100755 tests/common/test_classprecedence.py mode change 100644 => 100755 tests/common/test_datatypeconversion.py mode change 100644 => 100755 tests/docker-compose.yml mode change 100644 => 100755 tox.ini diff --git a/.dockerignore b/.dockerignore old mode 100644 new mode 100755 diff --git a/.github/dependabot.yml b/.github/dependabot.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/devskim-analysis.yml b/.github/workflows/devskim-analysis.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/agent.py b/agent.py old mode 100644 new mode 100755 diff --git a/api/__init__.py b/api/__init__.py old mode 100644 new mode 100755 diff --git a/api/custom_logging.py b/api/custom_logging.py old mode 100644 new mode 100755 diff --git a/api/server.py b/api/server.py old mode 100644 new mode 100755 diff --git a/cli/__init__.py b/cli/__init__.py old mode 100644 new mode 100755 diff --git a/cli/command.py b/cli/command.py old mode 100644 new mode 100755 diff --git a/common/__init__.py b/common/__init__.py old mode 100644 new mode 100755 diff --git a/common/config.json b/common/config.json old mode 100644 new mode 100755 diff --git a/doc/SDMX to JSON-LD.drawio b/doc/SDMX to JSON-LD.drawio old mode 100644 new mode 100755 diff --git a/doc/api.yaml b/doc/api.yaml old mode 100644 new mode 100755 diff --git a/docker/Dockerfile b/docker/Dockerfile old mode 100644 new mode 100755 diff --git a/docker/config.json b/docker/config.json old mode 100644 new mode 100755 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml old mode 100644 new mode 100755 diff --git a/examples/observation.ttl b/examples/observation.ttl old mode 100644 new mode 100755 diff --git a/examples/sep-dsd-2.ttl b/examples/sep-dsd-2.ttl old mode 100644 new mode 100755 diff --git a/examples/structures-accounts.ttl b/examples/structures-accounts.ttl old mode 100644 new mode 100755 diff --git a/examples/structures-tourism.ttl b/examples/structures-tourism.ttl old mode 100644 new mode 100755 diff --git a/images/logo.png b/images/logo.png old mode 100644 new mode 100755 diff --git a/poetry.lock b/poetry.lock old mode 100644 new mode 100755 diff --git a/pyproject.toml b/pyproject.toml old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index 9e5b0e2..e847606 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ # python3.10, virtualenv -ppython3.10 .venv -lark==1.1.3 +lark==1.1.5 secure==0.3.0 docopt==0.6.2 schema==0.7.5 hi-dateinfer==0.4.6 -fastapi==0.85.1 -uvicorn==0.19.0 +fastapi==0.88.0 +uvicorn==0.20.0 python-multipart==0.0.5 loguru==0.6.0 requests==2.28.1 diff --git a/sdmx2jsonld/README.md b/sdmx2jsonld/README.md old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/__init__.py b/sdmx2jsonld/__init__.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/config.py b/sdmx2jsonld/common/config.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/rdf.py b/sdmx2jsonld/common/rdf.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/common/regparser.py b/sdmx2jsonld/common/regparser.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/exceptions/__init__.py b/sdmx2jsonld/exceptions/__init__.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/grammar/grammar.lark b/sdmx2jsonld/grammar/grammar.lark old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/__init__.py b/sdmx2jsonld/transform/__init__.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/attribute.py b/sdmx2jsonld/transform/attribute.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py old mode 100644 new mode 100755 index 0046d80..3bcb7a4 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -91,7 +91,7 @@ def add_dataset(self, dataset_id): self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + hash1 # Add dataset id - self.data['dataset']['value'] = dataset_id + self.data['qb:dataset']['value'] = dataset_id def add_data(self, title, dataset_id, data): # We need to complete the data corresponding to the Catalogue: rdfs:label diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/context.py b/sdmx2jsonld/transform/context.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/dimension.py b/sdmx2jsonld/transform/dimension.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py old mode 100644 new mode 100755 diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py old mode 100644 new mode 100755 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/common/__init__.py b/tests/common/__init__.py old mode 100644 new mode 100755 diff --git a/tests/common/test_classprecedence.py b/tests/common/test_classprecedence.py old mode 100644 new mode 100755 diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py old mode 100644 new mode 100755 diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini old mode 100644 new mode 100755 From 83bbc7cb139b92377a79d677450d841ac75438dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 9 Jan 2023 13:48:56 +0100 Subject: [PATCH 08/74] Adding Observation class with Error control over confStatus values --- sdmx2jsonld/common/classprecedence.py | 18 +-- .../sdmxattributes/confirmationStatus.py | 48 ++++++ sdmx2jsonld/sdmxattributes/exceptions.py | 22 +++ .../sdmxattributes/observationStatus.py | 0 sdmx2jsonld/transform/catalogue.py | 18 ++- sdmx2jsonld/transform/entitytype.py | 10 +- sdmx2jsonld/transform/observation.py | 145 ++++++++++++++++++ tests/common/test_classprecedence.py | 10 +- tests/sdmxattributes/__init__.py | 0 .../sdmxattributes/test_confirmationStatus.py | 66 ++++++++ 10 files changed, 316 insertions(+), 21 deletions(-) create mode 100644 sdmx2jsonld/sdmxattributes/confirmationStatus.py create mode 100644 sdmx2jsonld/sdmxattributes/exceptions.py create mode 100644 sdmx2jsonld/sdmxattributes/observationStatus.py create mode 100644 sdmx2jsonld/transform/observation.py create mode 100644 tests/sdmxattributes/__init__.py create mode 100644 tests/sdmxattributes/test_confirmationStatus.py diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py index bf642c9..1876ca8 100755 --- a/sdmx2jsonld/common/classprecedence.py +++ b/sdmx2jsonld/common/classprecedence.py @@ -63,13 +63,13 @@ def precedence(self, data: list) -> str: # the same time a DimensionProperty and AttributeProperty, this is an ERROR and we need to report. result = all(element == 250 for element in classes_values) and len(data) > 1 if result is True: - raise ClassesPrecedencePropertyError(data) + raise ClassPrecedencePropertyError(data) # In case that we have several values identical of type Class, we need to report a WARNING message because maybe # it is not needed multitype in that case. result = all(element == 20 for element in classes_values) and len(data) > 1 if result is True: - raise ClassesPrecedenceClassError(data) + raise ClassPrecedenceClassError(data) # In other chase, we return the max value of the list aux = max(classes_values) @@ -78,7 +78,7 @@ def precedence(self, data: list) -> str: return aux -class ClassesPrecedenceError(Exception): +class ClassPrecedenceError(Exception): """Base class for other exceptions""" def __init__(self, data, message): @@ -89,12 +89,12 @@ def __str__(self): return f'{self.data} -> {self.message}' -class ClassesPrecedencePropertyError(ClassesPrecedenceError): +class ClassPrecedencePropertyError(ClassPrecedenceError): """Raised when the input value is too small""" - """Exception raised for errors in the input salary. + """Exception raised for errors in the input data. Attributes: - salary -- input salary which caused the error + data -- input data which caused the error message -- explanation of the error """ @@ -102,12 +102,12 @@ def __init__(self, data, message="Incompatible multiclass definition"): super().__init__(data=data, message=message) -class ClassesPrecedenceClassError(ClassesPrecedenceError): +class ClassPrecedenceClassError(ClassPrecedenceError): """Raised when the input value is too large""" - """Exception raised for errors in the input salary. + """Exception raised for errors in the input data. Attributes: - salary -- input salary which caused the error + data -- input data which caused the error message -- explanation of the error """ diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py new file mode 100644 index 0000000..5e4c473 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -0,0 +1,48 @@ +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassConfStatusError + + +class ConfStatus: + status: list() = [ + "F", + "N", + "C", + "D", + "S", + "A", + "O", + "T", + "G", + "M", + "E", + "P" + ] + + def fix_value(self, value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form confStatus-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + value_upper = value.upper() + + if value_upper in self.status: + return value_upper + else: + # we could receive a value in the format confStatus- + m = search('CONFSTATUS-(.*)', value_upper) + + if m is not None: + status = m.group(1) + + if status in self.status: + return status + else: + message = f"ConfStatus value is not included in the list of available values,\n" \ + f" got:{value}\n" \ + f" expected:{['confStatus-'+x for x in self.status]}" + + raise ClassConfStatusError(data=value, message=message) + + else: + # We received a value that it is not following the template format + raise ClassConfStatusError(value) diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/sdmxattributes/exceptions.py new file mode 100644 index 0000000..9aa4f55 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/exceptions.py @@ -0,0 +1,22 @@ +class ClassSDMXAttributeError(Exception): + """Base class for other exceptions""" + + def __init__(self, data, message): + self.message = message + self.data = data + + def __str__(self): + return f'{self.data} -> {self.message}' + + +class ClassConfStatusError(ClassSDMXAttributeError): + """Raised when the input value is not included in the list of available values for confStatus""" + """Exception raised for errors in the input data. + + Attributes: + data -- input data which caused the error + message -- explanation of the error + """ + + def __init__(self, data, message="ConfStatus value is not the expected"): + super().__init__(data=data, message=message) diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 3bcb7a4..d50d5d5 100755 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -36,7 +36,7 @@ def __init__(self): "id": str(), "type": "CatalogueDCAT-AP", "qb:dataset": { - "type": "object", + "type": "Relationship", "value": str() }, @@ -66,7 +66,7 @@ def __init__(self): }, "dct:title": { - "type": "Array", + "type": "Property", "value": list() }, @@ -99,13 +99,23 @@ def add_data(self, title, dataset_id, data): # Add the title key = self.keys['dct:title'] - self.data[key] = title + self.data[key]['value'] = title # Add the id self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + dataset_id + # Add the publisher + key = self.get_key(requested_key='dcterms:publisher') + position = data.index(key) + 1 + self.data['dct:publisher']['value'] = data[position][0] + + # Add structure + key = self.get_key(requested_key='qb:structure') + position = data.index(key) + 1 + self.data['qb:dataset']['value'] = data[position][0] + # Get the rest of the data - data = get_rest_data(data=data) + data = get_rest_data(data=data, not_allowed_keys=['rdfs:label']) # add the new data to the dataset structure self.patch_data(data, False) diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 161472c..76bb70e 100755 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -26,10 +26,11 @@ from sdmx2jsonld.transform.concept import Concept from sdmx2jsonld.transform.attribute import Attribute from sdmx2jsonld.transform.catalogue import CatalogueDCATAP +from sdmx2jsonld.transform.observation import Observation from logging import getLogger from datetime import datetime from sdmx2jsonld.common.regparser import RegParser -from sdmx2jsonld.common.classprecedence import Precedence, ClassesPrecedencePropertyError, ClassesPrecedenceClassError +from sdmx2jsonld.common.classprecedence import Precedence, ClassPrecedencePropertyError, ClassPrecedenceClassError logger = getLogger() @@ -59,6 +60,7 @@ def __init__(self): self.context = dict() self.context_mapping = dict() self.catalogue = CatalogueDCATAP() + self.observation = Observation() self.pre = Precedence() @@ -84,10 +86,10 @@ def __find_entity_type__(self, string): # We have two options, a well-know object list to be found in the self.entities or # the conceptList defined in the turtle file data = self.entities[data] - except ClassesPrecedencePropertyError as error: + except ClassPrecedencePropertyError as error: logger.error(str(error)) data = self.entities[data[0]] - except ClassesPrecedenceClassError as error: + except ClassPrecedenceClassError as error: logger.warning(str(error)) data = self.entities['rdfs:Class'] except KeyError: @@ -151,6 +153,8 @@ def create_data(self, type, data, title): print(identifier) elif type == 'Observation': identifier = parser.obtain_id(title) + self.observation.add_data(title=title, observation_id=identifier, data=data) + print(identifier) elif type == 'Dataset': identifier = parser.obtain_id(title) diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py new file mode 100644 index 0000000..34b56ea --- /dev/null +++ b/sdmx2jsonld/transform/observation.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +from logging import getLogger +from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus +from re import search + +logger = getLogger() + + +class Observation(CommonClass): + def __init__(self): + super().__init__(entity='Observation') + + self.data = { + "id": str(), + "type": "Observation", + "title": { + "type": "Property", + "value": str() + }, + "identifier": { + "type": "Property", + "value": str() + }, + "dataSet": { + "type": "Property", + "object": str() + }, + "confStatus": { + "type": "Property", + "value": str() + }, + "decimals": { + "type": "Property", + "value": int() + }, + "obsStatus": { + "type": "Property", + "value": str() + }, + "unitMult": { + "type": "Property", + "value": int() + }, + "freq": { + "type": "Property", + "value": str() + }, + "refArea": { + "type": "Property", + "value": str() + }, + "timePeriod": { + "type": "Property", + "value": str() + }, + "obsValue": { + "type": "Property", + "value": float() + }, + "dimensions": { + "type": "Property", + "value": list() + }, + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.SDMX/master/context.jsonld" + ] + } + + self.concept_id = str() + self.keys = {k: k for k in self.data.keys()} + + def add_data(self, title, observation_id, data): + # We have a list of dimensions, a dataset, a list of attributes, a list of dimensions attributes + # and an observation + + # Add the confStatus + key = self.__assign_property__(requested_key='sdmx-attribute:confStatus', data=data) + self.data[key]['value'] = ConfStatus().fix_value(value=self.data[key]['value']) + + # Add the id + self.data['id'] = "urn:ngsi-ld:Observation:" + observation_id + + # Add the decimals + self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) + + # Add obsStatus + self.__assign_property__(requested_key='sdmx-attribute:obsStatus', data=data) + + # Add unitMult + self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) + + def __assign_property__(self, requested_key, data): + key = self.get_key(requested_key=requested_key) + position = data.index(requested_key) + 1 + self.data[key]['value'] = data[position][0] + + return key + + def get_key(self, requested_key): + try: + key = self.keys[requested_key] + return key + except KeyError: + # The key did not exist therefore we add to the list with this value + # We need to check if it exists without prefix + m = search('(.*):(.*)', requested_key) + + if m is not None: + prefix = m.group(1) + subfix = m.group(2) + + try: + key = self.keys[subfix] + return key + except KeyError: + # Even subfix is not in the list, decide to add to the list of keys + self.keys[requested_key] = requested_key + return requested_key + else: + # Weird situation, it is an key without prefix and not recognised, decide to add it + self.keys[requested_key] = requested_key + + return requested_key diff --git a/tests/common/test_classprecedence.py b/tests/common/test_classprecedence.py index 974b3cf..a9347ef 100755 --- a/tests/common/test_classprecedence.py +++ b/tests/common/test_classprecedence.py @@ -20,7 +20,7 @@ # under the License. ## from unittest import TestCase -from sdmx2jsonld.common.classprecedence import Precedence, ClassesPrecedencePropertyError, ClassesPrecedenceClassError +from sdmx2jsonld.common.classprecedence import Precedence, ClassPrecedencePropertyError, ClassPrecedenceClassError class Test(TestCase): @@ -78,28 +78,28 @@ def test_precedence_classes_with_dimension_and_attribute_values(self): different from this, therefore we should return an error) 2) "rdfs:Class", "owl:Class" """ - with self.assertRaises(ClassesPrecedencePropertyError) as error: + with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:DimensionProperty", "qb:AttributeProperty"]) self.assertEqual(str(error.exception), "['qb:DimensionProperty', 'qb:AttributeProperty'] -> Incompatible multiclass definition") def test_precedence_classes_with_attribute_and_measure_values(self): - with self.assertRaises(ClassesPrecedencePropertyError) as error: + with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:AttributeProperty", "qb:MeasureProperty"]) self.assertEqual(str(error.exception), "['qb:AttributeProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition") def test_precedence_classes_with_dimension_and_measure_values(self): - with self.assertRaises(ClassesPrecedencePropertyError) as error: + with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:DimensionProperty", "qb:MeasureProperty"]) self.assertEqual(str(error.exception), "['qb:DimensionProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition") def test_precedence_classes_with_class_values(self): - with self.assertRaises(ClassesPrecedenceClassError) as error: + with self.assertRaises(ClassPrecedenceClassError) as error: _ = self.pre.precedence(["rdfs:Class", "owl:Class"]) self.assertEqual(str(error.exception), "['rdfs:Class', 'owl:Class'] -> Possible redundant Class definition") diff --git a/tests/sdmxattributes/__init__.py b/tests/sdmxattributes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py new file mode 100644 index 0000000..6c21816 --- /dev/null +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -0,0 +1,66 @@ +from unittest import TestCase +from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus +from sdmx2jsonld.sdmxattributes.exceptions import ClassConfStatusError + + +class TestConfStatus(TestCase): + def setUp(self): + self.conversion = ConfStatus() + + def test_fix_value_data_in_predefined_values(self): + value = 'F' + expected = 'F' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + value = 'f' + expected = 'F' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_fix_value_data_in_predefined_values_with_prefix(self): + value = 'confStatus-A' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + value = 'confstatus-a' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): + value = 'confStatus-EEEEE' + expected = "confStatus-EEEEE -> ConfStatus value is not included in the list of available values,\n" + \ + " got:confStatus-EEEEE\n" + \ + " expected:['confStatus-F', 'confStatus-N', 'confStatus-C', 'confStatus-D', 'confStatus-S', " \ + "'confStatus-A', 'confStatus-O', 'confStatus-T', 'confStatus-G', 'confStatus-M', 'confStatus-E', " \ + "'confStatus-P']" + + with self.assertRaises(ClassConfStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_fix_value_unexpected_value(self): + value = 'EEEE' + expected = 'Error...' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + value = 'lkdjlks-A' + expected = 'Error...' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nconfStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" From bca13e5f12eeab59f4fd65215191fedfb67b543f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 10 Jan 2023 18:16:46 +0100 Subject: [PATCH 09/74] Adding ObsStatus management and testing class --- sdmx2jsonld/sdmxattributes/exceptions.py | 13 ++++ .../sdmxattributes/observationStatus.py | 56 +++++++++++++++ sdmx2jsonld/transform/entitytype.py | 3 + sdmx2jsonld/transform/observation.py | 7 +- sdmx2jsonld/transform/parser.py | 2 + sdmx2jsonld/transform/transformer.py | 5 ++ .../sdmxattributes/test_confirmationStatus.py | 23 ++++--- .../sdmxattributes/test_observationStatus.py | 68 +++++++++++++++++++ 8 files changed, 165 insertions(+), 12 deletions(-) create mode 100644 tests/sdmxattributes/test_observationStatus.py diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/sdmxattributes/exceptions.py index 9aa4f55..c103216 100644 --- a/sdmx2jsonld/sdmxattributes/exceptions.py +++ b/sdmx2jsonld/sdmxattributes/exceptions.py @@ -20,3 +20,16 @@ class ClassConfStatusError(ClassSDMXAttributeError): def __init__(self, data, message="ConfStatus value is not the expected"): super().__init__(data=data, message=message) + + +class ClassObsStatusError(ClassSDMXAttributeError): + """Raised when the input value is not included in the list of available values for obsStatus""" + """Exception raised for errors in the input data. + + Attributes: + data -- input data which caused the error + message -- explanation of the error + """ + + def __init__(self, data, message="ObsStatus value is not the expected"): + super().__init__(data=data, message=message) diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index e69de29..8344e1f 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -0,0 +1,56 @@ +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassObsStatusError + + +class ObsStatus: + status: list() = [ + "A", + "B", + "D", + "E", + "F", + "G", + "I", + "K", + "W", + "O", + "M", + "P", + "S", + "L", + "H", + "Q", + "J", + "N", + "U", + "V" + ] + + def fix_value(self, value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form obsStatus-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + value_upper = value.upper() + + if value_upper in self.status: + return value_upper + else: + # we could receive a value in the format obsStatus- + m = search('OBSSTATUS-(.*)', value_upper) + + if m is not None: + status = m.group(1) + + if status in self.status: + return status + else: + message = f"ObsStatus value is not included in the list of available values,\n" \ + f" got:{value}\n" \ + f" expected:{['obsStatus-'+x for x in self.status]}" + + raise ClassObsStatusError(data=value, message=message) + + else: + # We received a value that it is not following the template format + raise ClassObsStatusError(value) diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 76bb70e..225097d 100755 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -216,6 +216,9 @@ def __get_subject__(self, title): def get_catalogue(self): return self.catalogue.get() + def get_observation(self): + return self.observation.get() + def get_dataset(self): return self.dataset.get() diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index 34b56ea..3b3c372 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -23,6 +23,7 @@ from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus +from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from re import search logger = getLogger() @@ -106,7 +107,8 @@ def add_data(self, title, observation_id, data): self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) # Add obsStatus - self.__assign_property__(requested_key='sdmx-attribute:obsStatus', data=data) + key = self.__assign_property__(requested_key='sdmx-attribute:obsStatus', data=data) + self.data[key]['value'] = ObsStatus().fix_value(value=self.data[key]['value']) # Add unitMult self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) @@ -143,3 +145,6 @@ def get_key(self, requested_key): self.keys[requested_key] = requested_key return requested_key + + def get(self): + return self.data diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 7f7ebd0..591921b 100755 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -85,6 +85,7 @@ def parsing_file(self, content: TextIOWrapper, out: bool): elif content is not None: print() pprint(transform.get_catalogue()) + pprint(transform.get_observation()) pprint(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] @@ -103,6 +104,7 @@ def parsing_string(self, content: StringIO): # Serializing json payload result = list() result.append(transform.get_catalogue()) + result.append(transform.get_observation()) result.append(transform.get_dataset()) [result.append(x.get()) for x in transform.get_dimensions()] [result.append(x.get()) for x in transform.get_attributes()] diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index a650944..a1062a2 100755 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -115,6 +115,9 @@ def get_context_mapping(self): def get_catalogue(self): return self.entity_type.get_catalogue() + def get_observation(self): + return self.entity_type.get_observation() + def get_dataset(self): return self.entity_type.get_dataset() @@ -133,6 +136,8 @@ def get_conceptLists(self): def save(self): self.entity_type.save('catalogue') + self.entity_type.save('observation') + if self.entity_type.dataset.data['id'] != '': self.entity_type.save('dataset') diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py index 6c21816..ae0d2d5 100644 --- a/tests/sdmxattributes/test_confirmationStatus.py +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -50,17 +50,18 @@ def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): self.assertEqual(str(error.exception), expected) - def test_fix_value_unexpected_value(self): + def test_fix_value_unexpected_value_without_prefix(self): value = 'EEEE' - expected = 'Error...' - obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + expected = 'EEEE -> ConfStatus value is not the expected' + with self.assertRaises(ClassConfStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + def test_fix_value_unexpected_prefix(self): value = 'lkdjlks-A' - expected = 'Error...' - obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + expected = 'lkdjlks-A -> ConfStatus value is not the expected' + with self.assertRaises(ClassConfStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) diff --git a/tests/sdmxattributes/test_observationStatus.py b/tests/sdmxattributes/test_observationStatus.py new file mode 100644 index 0000000..ecbdf0a --- /dev/null +++ b/tests/sdmxattributes/test_observationStatus.py @@ -0,0 +1,68 @@ +from unittest import TestCase +from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus +from sdmx2jsonld.sdmxattributes.exceptions import ClassObsStatusError + + +class TestObsStatus(TestCase): + def setUp(self): + self.conversion = ObsStatus() + + def test_fix_value_data_in_predefined_values(self): + value = 'A' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nobsStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + value = 'a' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nobsStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_fix_value_data_in_predefined_values_with_prefix(self): + value = 'obsStatus-A' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nobsStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + value = 'obsstatus-a' + expected = 'A' + obtained = self.conversion.fix_value(value=value) + assert obtained == expected, f"\nobsStatus was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): + value = 'obsStatus-EEEEE' + expected = "obsStatus-EEEEE -> ObsStatus value is not included in the list of available values,\n" + \ + " got:obsStatus-EEEEE\n" + \ + " expected:['obsStatus-A', 'obsStatus-B', 'obsStatus-D', 'obsStatus-E', 'obsStatus-F', " \ + "'obsStatus-G', 'obsStatus-I', 'obsStatus-K', 'obsStatus-W', 'obsStatus-O', 'obsStatus-M', " \ + "'obsStatus-P', 'obsStatus-S', 'obsStatus-L', 'obsStatus-H', 'obsStatus-Q', 'obsStatus-J', " \ + "'obsStatus-N', 'obsStatus-U', 'obsStatus-V']" + + with self.assertRaises(ClassObsStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_fix_value_unexpected_value_without_prefix(self): + value = 'EEEE' + expected = 'EEEE -> ObsStatus value is not the expected' + with self.assertRaises(ClassObsStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_fix_value_unexpected_prefix(self): + value = 'lkdjlks-A' + expected = 'lkdjlks-A -> ObsStatus value is not the expected' + with self.assertRaises(ClassObsStatusError) as error: + _ = self.conversion.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) From b0ff0d09bd1e56705f3a8e268071f237bea998c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 12 Jan 2023 17:19:22 +0100 Subject: [PATCH 10/74] Adding control of Decimal and UnitMult codes with unittests --- sdmx2jsonld/sdmxattributes/code.py | 58 ++++++++++++ sdmx2jsonld/sdmxattributes/exceptions.py | 13 +++ sdmx2jsonld/transform/observation.py | 7 +- tests/sdmxattributes/test_code.py | 111 +++++++++++++++++++++++ 4 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 sdmx2jsonld/sdmxattributes/code.py create mode 100644 tests/sdmxattributes/test_code.py diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py new file mode 100644 index 0000000..9836b5b --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -0,0 +1,58 @@ +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassCode + + +class Code: + status: list() + type: str() + data_range: list() + + def __init__(self, typecode): + self.typecode = typecode + + if typecode == "decimals": + self.data_range = range(0, 15) + elif typecode == "unitMult": + self.data_range = range(0, 13) + + def fix_value(self, value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form obsStatus-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + m = search(f'sdmx-code:{self.typecode}-(.*)', str(value)) + + if m is not None: + number = int(m.group(1)) + + if number not in self.data_range: + raise ClassCode(data=value, + message=f'{self.typecode} out of range, got: {number} {self.data_range}') + else: + # The data is not following the sdmx-code: we have to check which one + # 1) Check if there is a value without the prefix + m = search(f'{self.typecode}-(.*)', str(value)) + + if m is not None: + number = int(m.group(1)) + + if number not in self.data_range: + raise ClassCode(data=value, + message=f'{self.typecode} out of range, got: {number} {self.data_range}') + else: + # We need to check is there is an integer number between a valid range + if isinstance(value, int): + # Need to check the range + number = value + if number not in self.data_range: + raise ClassCode(data=value, + message=f'{self.typecode} out of range, got: {number} {self.data_range}') + elif isinstance(value, str): + number = int(value) + if number not in self.data_range: + raise ClassCode(data=value, + message=f'{self.typecode} out of range, got: {number} {self.data_range}') + else: + print("Error") + + return number diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/sdmxattributes/exceptions.py index c103216..2dba1da 100644 --- a/sdmx2jsonld/sdmxattributes/exceptions.py +++ b/sdmx2jsonld/sdmxattributes/exceptions.py @@ -33,3 +33,16 @@ class ClassObsStatusError(ClassSDMXAttributeError): def __init__(self, data, message="ObsStatus value is not the expected"): super().__init__(data=data, message=message) + + +class ClassCode(ClassSDMXAttributeError): + """Raised when the input value is not included in the list of available values for obsStatus""" + """Exception raised for errors in the input data. + + Attributes: + data -- input data which caused the error + message -- explanation of the error + """ + + def __init__(self, data, message="Decimals value is not the expected"): + super().__init__(data=data, message=message) diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index 3b3c372..8ebe0de 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -24,6 +24,7 @@ from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus +from sdmx2jsonld.sdmxattributes.code import Code from re import search logger = getLogger() @@ -104,14 +105,16 @@ def add_data(self, title, observation_id, data): self.data['id'] = "urn:ngsi-ld:Observation:" + observation_id # Add the decimals - self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) + key = self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) + self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) # Add obsStatus key = self.__assign_property__(requested_key='sdmx-attribute:obsStatus', data=data) self.data[key]['value'] = ObsStatus().fix_value(value=self.data[key]['value']) # Add unitMult - self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) + key = self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) + self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) def __assign_property__(self, requested_key, data): key = self.get_key(requested_key=requested_key) diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py new file mode 100644 index 0000000..967c524 --- /dev/null +++ b/tests/sdmxattributes/test_code.py @@ -0,0 +1,111 @@ +from unittest import TestCase +from sdmx2jsonld.sdmxattributes.code import Code +from sdmx2jsonld.sdmxattributes.exceptions import ClassCode + + +class TestConfStatus(TestCase): + def test_code_value_with_prefix(self): + value = 'sdmx-code:decimals-1' + expected = 1 + + code = Code(typecode="decimals") + obtained = code.fix_value(value=value) + assert obtained == expected, f"\ncode was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_code_negative_value_with_prefix(self): + value = 'sdmx-code:decimals--1' + expected = 'sdmx-code:decimals--1 -> decimals out of range, got: -1 range(0, 15)' + + code = Code(typecode="decimals") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_code_value_bigger_than_maximum_with_prefix(self): + value = 'sdmx-code:decimals-67' + expected = 'sdmx-code:decimals-67 -> decimals out of range, got: 67 range(0, 15)' + + code = Code(typecode="decimals") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_code_value_without_prefix(self): + value = 'unitMult-1' + expected = 1 + + code = Code(typecode="unitMult") + obtained = code.fix_value(value=value) + assert obtained == expected, f"\ncode was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_code_negative_value_without_prefix(self): + value = 'unitMult--1' + expected = 'unitMult--1 -> unitMult out of range, got: -1 range(0, 13)' + + code = Code(typecode="unitMult") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_code_value_bigger_than_maximum_without_prefix(self): + value = 'unitMult-67' + expected = 'unitMult-67 -> unitMult out of range, got: 67 range(0, 13)' + + code = Code(typecode="unitMult") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_code_integer_value(self): + value = 2 + expected = 2 + + code = Code(typecode="unitMult") + obtained = code.fix_value(value=value) + assert obtained == expected, f"\ncode was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_code_integer_value_out_of_range(self): + value = 25 + expected = '25 -> unitMult out of range, got: 25 range(0, 13)' + + code = Code(typecode="unitMult") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) + + def test_code_string_value(self): + value = '2' + expected = 2 + + code = Code(typecode="unitMult") + obtained = code.fix_value(value=value) + assert obtained == expected, f"\ncode was not the expected," \ + f"\n got : {obtained}" \ + f"\n expected: {expected}" + + def test_code_string_value_out_of_range(self): + value = '25' + expected = '25 -> unitMult out of range, got: 25 range(0, 13)' + + code = Code(typecode="unitMult") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) From 15afb5d3814e8542d5669c2d61154ec781071d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 12 Jan 2023 17:27:03 +0100 Subject: [PATCH 11/74] Improve Code class redufing redundant code --- sdmx2jsonld/sdmxattributes/code.py | 26 ++++++++++---------------- tests/sdmxattributes/test_code.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 9836b5b..839f4ca 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -20,14 +20,12 @@ def fix_value(self, value): # then maybe could be in the form obsStatus-, so we have to extract the substring and # return that substring if it is in the list of values, if not return an error. # any other value will return an error + number: int() = 0 + m = search(f'sdmx-code:{self.typecode}-(.*)', str(value)) if m is not None: number = int(m.group(1)) - - if number not in self.data_range: - raise ClassCode(data=value, - message=f'{self.typecode} out of range, got: {number} {self.data_range}') else: # The data is not following the sdmx-code: we have to check which one # 1) Check if there is a value without the prefix @@ -35,24 +33,20 @@ def fix_value(self, value): if m is not None: number = int(m.group(1)) - - if number not in self.data_range: - raise ClassCode(data=value, - message=f'{self.typecode} out of range, got: {number} {self.data_range}') else: # We need to check is there is an integer number between a valid range if isinstance(value, int): # Need to check the range number = value - if number not in self.data_range: - raise ClassCode(data=value, - message=f'{self.typecode} out of range, got: {number} {self.data_range}') elif isinstance(value, str): - number = int(value) - if number not in self.data_range: + try: + number = int(value) + except ValueError: raise ClassCode(data=value, - message=f'{self.typecode} out of range, got: {number} {self.data_range}') - else: - print("Error") + message=f'Data is not a valid value') + + if number not in self.data_range: + raise ClassCode(data=value, + message=f'{self.typecode} out of range, got: {number} {self.data_range}') return number diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py index 967c524..08f0620 100644 --- a/tests/sdmxattributes/test_code.py +++ b/tests/sdmxattributes/test_code.py @@ -109,3 +109,14 @@ def test_code_string_value_out_of_range(self): _ = code.fix_value(value=value) self.assertEqual(str(error.exception), expected) + + def test_any_other_code(self): + value = 'sadf' + expected = 'sadf -> Data is not a valid value' + + code = Code(typecode="unitMult") + + with self.assertRaises(ClassCode) as error: + _ = code.fix_value(value=value) + + self.assertEqual(str(error.exception), expected) From 497b092a9039b20879571178fece358b53ae4987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 13 Jan 2023 16:47:03 +0100 Subject: [PATCH 12/74] Adding Frequency management and correction other issues --- sdmx2jsonld/sdmxattributes/code.py | 4 +-- sdmx2jsonld/sdmxattributes/exceptions.py | 15 +++++++++++- sdmx2jsonld/sdmxattributes/frequency.py | 20 +++++++++++++++ sdmx2jsonld/transform/observation.py | 31 +++++++++++++++++++++++- 4 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 sdmx2jsonld/sdmxattributes/frequency.py diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 839f4ca..51e651e 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -17,8 +17,8 @@ def __init__(self, typecode): def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value - # then maybe could be in the form obsStatus-, so we have to extract the substring and - # return that substring if it is in the list of values, if not return an error. + # then maybe could be in the form decimals- or unitMult-, so we have to extract + # the substring and return that substring if it is in the list of values, if not return an error. # any other value will return an error number: int() = 0 diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/sdmxattributes/exceptions.py index 2dba1da..83aa60a 100644 --- a/sdmx2jsonld/sdmxattributes/exceptions.py +++ b/sdmx2jsonld/sdmxattributes/exceptions.py @@ -36,7 +36,20 @@ def __init__(self, data, message="ObsStatus value is not the expected"): class ClassCode(ClassSDMXAttributeError): - """Raised when the input value is not included in the list of available values for obsStatus""" + """Raised when the input value is not included in the list of available values for unitMult and decimals""" + """Exception raised for errors in the input data. + + Attributes: + data -- input data which caused the error + message -- explanation of the error + """ + + def __init__(self, data, message="Decimals value is not the expected"): + super().__init__(data=data, message=message) + + +class ClassFreqError(ClassSDMXAttributeError): + """Raised when the input value is not included in the list of available values for Freq""" """Exception raised for errors in the input data. Attributes: diff --git a/sdmx2jsonld/sdmxattributes/frequency.py b/sdmx2jsonld/sdmxattributes/frequency.py new file mode 100644 index 0000000..4c56206 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/frequency.py @@ -0,0 +1,20 @@ +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError + + +class Frequency: + def fix_value(self, value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form freq-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + value_upper = value.upper() + + m = search('FREQ-(.*)', value_upper) + + if m is not None: + status = m.group(1) + return status + else: + # We received a value that it is not following the template format + raise ClassFreqError(value) diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index 8ebe0de..bfb9b90 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -25,6 +25,7 @@ from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from sdmx2jsonld.sdmxattributes.code import Code +from sdmx2jsonld.sdmxattributes.frequency import Frequency from re import search logger = getLogger() @@ -103,6 +104,7 @@ def add_data(self, title, observation_id, data): # Add the id self.data['id'] = "urn:ngsi-ld:Observation:" + observation_id + self.data['identifier']['value'] = observation_id # Add the decimals key = self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) @@ -116,10 +118,25 @@ def add_data(self, title, observation_id, data): key = self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) + # Add freq "pattern": "^_[OUZ]|[SQBNI]|OA|OM|[AMWDH]_*[0-9]*$" + # TODO: Add verification of coded data following the pattern + key = self.__assign_property__(requested_key='sdmx-dimension:freq', data=data) + self.data[key]['value'] = Frequency().fix_value(value=self.data[key]['value']) + + # Add reference Area + # TODO: Add verification of coded data following ISO-2, ISO-3, or M49 code + _ = self.__assign_property__(requested_key='sdmx-dimension:refArea', data=data) + + # Add timePeriod + _ = self.__assign_property__(requested_key='sdmx-dimension:timePeriod', data=data) + + # Add obsValue + _ = self.__assign_property__(requested_key='sdmx-measure:obsValue', data=data) + def __assign_property__(self, requested_key, data): key = self.get_key(requested_key=requested_key) position = data.index(requested_key) + 1 - self.data[key]['value'] = data[position][0] + self.data[key]['value'] = self.get_data(data[position]) return key @@ -151,3 +168,15 @@ def get_key(self, requested_key): def get(self): return self.data + + def get_data(self, data): + result = data + if isinstance(data, list): + result = self.get_data(data[0]) + + if isinstance(result, str): + m = search('"+(.*)"+', result) + if m is not None: + result = m.group(1) + + return result From 981f297fb5793491d50f89b1d82319245f67e3d8 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 17 Jan 2023 16:05:36 +0100 Subject: [PATCH 13/74] changes on datatypeconversion --- sdmx2jsonld/common/datatypeconversion.py | 2 +- tests/common/test_datatypeconversion.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index 0128405..45fa804 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -29,7 +29,7 @@ class DataTypeConversion: def __init__(self): self.types = { - 'xsd:dateTime': 'stoutc', + 'xsd:dateTime': 'stodt', 'xsd:int': 'stoi', 'xsd:boolean': 'stob' } diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index a2cee8f..3fa2bb6 100644 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -61,16 +61,15 @@ def test_string_to_dates(self): "2021-07-01T11:50:37.3", "2021-09-28T15:31:24.05", "Mon Jan 13 09:52:52 MST 2014", "Thu Jun 02 11:56:53 CDT 2011", "2022-12-12T10:00:00", "2022-05-11T10:00:00", - "Tue Dec 13 11:00:00 K 2022") + "Tue Dec 13 11:00:00 K 2022", "2021-07-01T11:58:08.642000") expected = ("2022-01-15T08:00:00+00:00", "2022-01-10T08:00:00+00:00", "2021-07-01T09:50:37.300000+00:00", "2021-09-28T13:31:24.050000+00:00", "2014-01-13T16:52:52+00:00", "2011-06-02T16:56:53+00:00", "2022-12-12T09:00:00+00:00", "2022-05-11T08:00:00+00:00", - "2022-12-13T01:00:00+00:00") - + "2022-12-13T01:00:00+00:00", "2021-07-01T09:58:08.642000+00:00") + # "2021-07-01T09:58:08.642000+00:00" d = zip(dates, expected) for test_date, expected_date in d: - # print(test_date) - # print(dtc.convert(test_date, token_type)) + print(test_date, " | ", dtc.convert(test_date, token_type)) assert (expected_date == dtc.convert(test_date, token_type)) \ No newline at end of file From f8d9237ea27c9f3f7fc8562caa25431c818cffc1 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 17 Jan 2023 16:31:25 +0100 Subject: [PATCH 14/74] updated datatypeconversion to fit input data again --- sdmx2jsonld/common/datatypeconversion.py | 7 ++++--- tests/common/test_datatypeconversion.py | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index 45fa804..cff1b16 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -29,7 +29,7 @@ class DataTypeConversion: def __init__(self): self.types = { - 'xsd:dateTime': 'stodt', + 'xsd:dateTime': 'stoutc', 'xsd:int': 'stoi', 'xsd:boolean': 'stob' } @@ -67,7 +67,7 @@ def stodt(value): else: raise Exception(f'Invalid format received: {type(value)}') - # result = self.correct_datatype_format(result) + result = self.correct_datatype_format(result) # print(f'format {result}') result = datetime.strptime(value, result).replace(tzinfo=timezone.utc).isoformat() @@ -105,7 +105,8 @@ def stob(value): raise Exception(f'Invalid value for boolean conversion: {str(value)}') try: - function = self.types[datatype] + f'(value="{data}")' + # jicg - function = self.types[datatype] + f'(value="{data}")' + function = self.types[datatype] + '(value=' + data + ')' return eval(function) except KeyError: # logger.error(f'Datatype not defined: {datatype}') diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index 3fa2bb6..3d3d1b1 100644 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -57,11 +57,11 @@ def test_string_to_dates(self): dtc = DataTypeConversion() token_type = 'xsd:dateTime' - dates = ("2022-01-15T08:00:00.000+00:00", "2022-01-10T09:00:00.000", - "2021-07-01T11:50:37.3", "2021-09-28T15:31:24.05", - "Mon Jan 13 09:52:52 MST 2014", "Thu Jun 02 11:56:53 CDT 2011", - "2022-12-12T10:00:00", "2022-05-11T10:00:00", - "Tue Dec 13 11:00:00 K 2022", "2021-07-01T11:58:08.642000") + dates = ('"2022-01-15T08:00:00.000+00:00"', '"2022-01-10T09:00:00.000"', + '"2021-07-01T11:50:37.3"', '"2021-09-28T15:31:24.05"', + '"Mon Jan 13 09:52:52 MST 2014"', '"Thu Jun 02 11:56:53 CDT 2011"', + '"2022-12-12T10:00:00"', '"2022-05-11T10:00:00"', + '"Tue Dec 13 11:00:00 K 2022"', '"2021-07-01T11:58:08.642000"') expected = ("2022-01-15T08:00:00+00:00", "2022-01-10T08:00:00+00:00", "2021-07-01T09:50:37.300000+00:00", "2021-09-28T13:31:24.050000+00:00", "2014-01-13T16:52:52+00:00", "2011-06-02T16:56:53+00:00", From 21b5d4cda4ffbbdf8c384a5b08c0f3e1f64cf7df Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 19 Jan 2023 13:46:08 +0100 Subject: [PATCH 15/74] Modified structures generated to be compliant with ngsi-ld --- agent.py | 1 + ngsild/__init__.py | 0 ngsild/ngsild_connector.py | 69 ++++++++++++++++++++++++++++++ sdmx2jsonld/transform/catalogue.py | 8 ++-- sdmx2jsonld/transform/concept.py | 7 ++- tests/test1.py | 19 ++++++++ 6 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 ngsild/__init__.py create mode 100644 ngsild/ngsild_connector.py create mode 100644 tests/test1.py diff --git a/agent.py b/agent.py index a56f3fb..fb22fdc 100644 --- a/agent.py +++ b/agent.py @@ -24,6 +24,7 @@ from sdmx2jsonld.transform.parser import Parser from api.server import launch from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken +from ngsild.ngsild_connector import NGSILDConnector if __name__ == '__main__': args = parse_cli() diff --git a/ngsild/__init__.py b/ngsild/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py new file mode 100644 index 0000000..ab07113 --- /dev/null +++ b/ngsild/ngsild_connector.py @@ -0,0 +1,69 @@ +from pathlib import Path +import json +from requests import post, exceptions + + +class NGSILDConnector: + def __init__(self, path=None): + if path == None: + config_path = Path.cwd().joinpath('common/config.json') + else: + config_path = Path.cwd().joinpath(path) + config = dict() + with open(config_path) as config_file: + config = json.load(config_file) + self.base_url = config['broker'] + + def get_url(self): + url = f"{self.base_url}/ngsi-ld/v1/entityOperations/create" + return url + + def send_pretty_much_data(self, json_object): + d = json.loads(json_object) + d = d if type(d) is list else [d] + + for elem in d: + rc, r = c.send_data(json.dumps(elem)) + print("Code: ", rc) + print(r) + + def send_data(self, json_object): + # Send the data to a FIWARE Context Broker instance + headers = { + 'Content-Type': 'application/ld+json', + 'Accept': 'application/ld+json' + } + + url = self.get_url() + resp = "..." + + r = post(url=url, headers=headers, data=json_object, timeout=5) + + resp = json.loads(r.text) + response_status_code = r.status_code + + # Let exceptions raise.... They can be controlled somewhere else. + return response_status_code, resp + + +from sdmx2jsonld.transform.parser import Parser +from io import StringIO + +if __name__ == "__main__": + c = NGSILDConnector('../common/config.json') + print(c.get_url()) + + parser = Parser() + with open("../examples/structures-tourism.ttl", "r") as rf: + rdf_data = rf.read() + + r = parser.parsing(StringIO(rdf_data), out=False) + + with open("/tmp/ww", "w") as f: + f.write(r) + + # print(data) + c.send_pretty_much_data(r) + # rc, r = c.send_data(data) + # print("Code: ", rc) + # print(r) diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7502ee0..bdf24b7 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -35,8 +35,8 @@ def __init__(self): "id": str(), "type": "CatalogueDCAT-AP", "dataset": { - "type": "object", - "value": str() + "type": "Relationship", + "object": str() }, "language": { @@ -65,7 +65,7 @@ def __init__(self): }, "title": { - "type": "Array", + "type": "Property", "value": list() }, @@ -89,7 +89,7 @@ def add_dataset(self, dataset_id): self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + hash1 # Add dataset id - self.data['dataset']['value'] = dataset_id + self.data['dataset']['object'] = dataset_id def get(self): return self.data diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index db3b126..951e9ee 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -41,7 +41,7 @@ def __init__(self): }, "skos:inScheme": { "type": "Relationship", - "value": str() + "object": str() }, "rdfs:subClassOf": { "type": "Property", @@ -170,7 +170,10 @@ def need_add_in_scheme(self, data): parser = RegParser() concept_schema = data[position][0] concept_schema = "urn:ngsi-ld:ConceptSchema:" + parser.obtain_id(concept_schema) - self.data['skos:inScheme']['value'] = concept_schema + if self.data['skos:inScheme']['type'] == 'Relationship': + self.data['skos:inScheme']['object'] = concept_schema + else: + self.data['skos:inScheme']['value'] = concept_schema def need_add_notation(self, data): try: diff --git a/tests/test1.py b/tests/test1.py new file mode 100644 index 0000000..288d4a9 --- /dev/null +++ b/tests/test1.py @@ -0,0 +1,19 @@ +from sdmx2jsonld.transform.parser import Parser +from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken + +if __name__ == "__main__": + # file_in = open("../examples/structures-accounts.ttl") + file_in = open("kex/e1.ttl") + generate_files = True + + # Start parsing the file + my_parser = Parser() + + try: + my_parser.parsing(content=file_in, out=generate_files) + except UnexpectedToken as e: + print(e) + except UnexpectedInput as e: + print(e) + except UnexpectedEOF as e: + print(e) From 3952189d4372ebb845dfa250428359db3073cb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 23 Jan 2023 12:29:16 +0100 Subject: [PATCH 16/74] Resolve concept.py error with type relationship and add more functionality to observation class --- sdmx2jsonld/transform/concept.py | 4 ++-- sdmx2jsonld/transform/observation.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index db3b126..f3f5db7 100755 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -41,7 +41,7 @@ def __init__(self): }, "skos:inScheme": { "type": "Relationship", - "value": str() + "object": str() }, "rdfs:subClassOf": { "type": "Property", @@ -170,7 +170,7 @@ def need_add_in_scheme(self, data): parser = RegParser() concept_schema = data[position][0] concept_schema = "urn:ngsi-ld:ConceptSchema:" + parser.obtain_id(concept_schema) - self.data['skos:inScheme']['value'] = concept_schema + self.data['skos:inScheme']['object'] = concept_schema def need_add_notation(self, data): try: diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index bfb9b90..67c2de7 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -26,6 +26,7 @@ from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from sdmx2jsonld.sdmxattributes.code import Code from sdmx2jsonld.sdmxattributes.frequency import Frequency +from sdmx2jsonld.common.regparser import RegParser from re import search logger = getLogger() @@ -47,7 +48,7 @@ def __init__(self): "value": str() }, "dataSet": { - "type": "Property", + "type": "Relationship", "object": str() }, "confStatus": { @@ -106,6 +107,9 @@ def add_data(self, title, observation_id, data): self.data['id'] = "urn:ngsi-ld:Observation:" + observation_id self.data['identifier']['value'] = observation_id + # Add title, the url string as it is in the turtle document + self.data['title']['value'] = title + # Add the decimals key = self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) @@ -133,10 +137,16 @@ def add_data(self, title, observation_id, data): # Add obsValue _ = self.__assign_property__(requested_key='sdmx-measure:obsValue', data=data) - def __assign_property__(self, requested_key, data): + # Add dataset + parser = RegParser() + key = self.__assign_property__(requested_key='qb:dataSet', data=data, key_property='object') + identifier = parser.obtain_id(self.data[key]['object']) + self.data[key]['object'] = 'urn:ngsi-ld:CatalogueDCAT-AP:' + identifier + + def __assign_property__(self, requested_key, data, key_property='value'): key = self.get_key(requested_key=requested_key) position = data.index(requested_key) + 1 - self.data[key]['value'] = self.get_data(data[position]) + self.data[key][key_property] = self.get_data(data[position]) return key From 5bc4692e963d94f6b3fc2257d40f5b53b8d4f5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 23 Jan 2023 15:58:54 +0100 Subject: [PATCH 17/74] Normalising the rest of pending data in catalogue.py --- sdmx2jsonld/transform/catalogue.py | 31 +++++++++++++++++++++--------- sdmx2jsonld/transform/dataset.py | 2 +- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index d50d5d5..7420e8f 100755 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -68,13 +68,7 @@ def __init__(self): "dct:title": { "type": "Property", "value": list() - }, - - - "@context": [ - "https://raw.githubusercontent.com/SEMICeu/DCAT-AP/master/releases/1.1/dcat-ap_1.1.jsonld", - "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" - ] + } } self.concept_id = str() @@ -115,18 +109,37 @@ def add_data(self, title, dataset_id, data): self.data['qb:dataset']['value'] = data[position][0] # Get the rest of the data - data = get_rest_data(data=data, not_allowed_keys=['rdfs:label']) + data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher']) # add the new data to the dataset structure self.patch_data(data, False) + # Add context + context = { + "@context": [ + "https://raw.githubusercontent.com/SEMICeu/DCAT-AP/master/releases/1.1/dcat-ap_1.1.jsonld", + "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" + ] + } + self.data.update(context) + def patch_data(self, data, language_map): if language_map: self.__complete_label__(title="Not specified", data=data) else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [self.data.update({k: v}) for k, v in data.items()] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] + + def __generate_property__(self, key, value): + result = { + key: { + "type": "Property", + "value": value + } + } + + return result def __complete_label__(self, title, data): try: diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 15cc49b..5a0a1c8 100755 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -121,7 +121,7 @@ def add_components(self, context, component): except ValueError: logger.error(f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}") - # Simplify Context amd order keys. It is possible that we call add_component before the dataset has been created + # Simplify Context and order keys. It is possible that we call add_component before the dataset has been created # therefore we need to add the corresponding context to the dataset if len(self.data['@context']) == 0: self.data['@context'] = context['@context'] From e1cf26f116b9d2cde1dbcbd2a8ec11b67eddbd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 25 Jan 2023 18:05:15 +0100 Subject: [PATCH 18/74] Translate __generate_property__ from catalogue to commonclass --- .dockerignore | 0 .github/dependabot.yml | 0 .github/workflows/devskim-analysis.yml | 0 .gitignore | 0 LICENSE | 0 README.md | 0 agent.py | 0 api/__init__.py | 0 api/custom_logging.py | 0 api/server.py | 0 cli/__init__.py | 0 cli/command.py | 0 common/__init__.py | 0 common/config.json | 0 doc/API.md | 0 docker/Dockerfile | 0 docker/config.json | 0 docker/docker-compose.yml | 0 examples/observation.ttl | 0 examples/sep-dsd-2.ttl | 0 examples/structures-accounts.ttl | 0 examples/structures-tourism.ttl | 0 sdmx2jsonld/common/commonclass.py | 10 ++++++++++ sdmx2jsonld/transform/catalogue.py | 10 ---------- 24 files changed, 10 insertions(+), 10 deletions(-) mode change 100755 => 100644 .dockerignore mode change 100755 => 100644 .github/dependabot.yml mode change 100755 => 100644 .github/workflows/devskim-analysis.yml mode change 100755 => 100644 .gitignore mode change 100755 => 100644 LICENSE mode change 100755 => 100644 README.md mode change 100755 => 100644 agent.py mode change 100755 => 100644 api/__init__.py mode change 100755 => 100644 api/custom_logging.py mode change 100755 => 100644 api/server.py mode change 100755 => 100644 cli/__init__.py mode change 100755 => 100644 cli/command.py mode change 100755 => 100644 common/__init__.py mode change 100755 => 100644 common/config.json mode change 100644 => 100755 doc/API.md mode change 100755 => 100644 docker/Dockerfile mode change 100755 => 100644 docker/config.json mode change 100755 => 100644 docker/docker-compose.yml mode change 100755 => 100644 examples/observation.ttl mode change 100755 => 100644 examples/sep-dsd-2.ttl mode change 100755 => 100644 examples/structures-accounts.ttl mode change 100755 => 100644 examples/structures-tourism.ttl diff --git a/.dockerignore b/.dockerignore old mode 100755 new mode 100644 diff --git a/.github/dependabot.yml b/.github/dependabot.yml old mode 100755 new mode 100644 diff --git a/.github/workflows/devskim-analysis.yml b/.github/workflows/devskim-analysis.yml old mode 100755 new mode 100644 diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/LICENSE b/LICENSE old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/agent.py b/agent.py old mode 100755 new mode 100644 diff --git a/api/__init__.py b/api/__init__.py old mode 100755 new mode 100644 diff --git a/api/custom_logging.py b/api/custom_logging.py old mode 100755 new mode 100644 diff --git a/api/server.py b/api/server.py old mode 100755 new mode 100644 diff --git a/cli/__init__.py b/cli/__init__.py old mode 100755 new mode 100644 diff --git a/cli/command.py b/cli/command.py old mode 100755 new mode 100644 diff --git a/common/__init__.py b/common/__init__.py old mode 100755 new mode 100644 diff --git a/common/config.json b/common/config.json old mode 100755 new mode 100644 diff --git a/doc/API.md b/doc/API.md old mode 100644 new mode 100755 diff --git a/docker/Dockerfile b/docker/Dockerfile old mode 100755 new mode 100644 diff --git a/docker/config.json b/docker/config.json old mode 100755 new mode 100644 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml old mode 100755 new mode 100644 diff --git a/examples/observation.ttl b/examples/observation.ttl old mode 100755 new mode 100644 diff --git a/examples/sep-dsd-2.ttl b/examples/sep-dsd-2.ttl old mode 100755 new mode 100644 diff --git a/examples/structures-accounts.ttl b/examples/structures-accounts.ttl old mode 100755 new mode 100644 diff --git a/examples/structures-tourism.ttl b/examples/structures-tourism.ttl old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index 3aea878..3b19013 100755 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -83,3 +83,13 @@ def generate_id(self, value, entity=None): aux = "urn:ngsi-ld:" + entity + ":" + aux return aux + + def __generate_property__(self, key, value): + result = { + key: { + "type": "Property", + "value": value + } + } + + return result diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7420e8f..c7c1874 100755 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -131,16 +131,6 @@ def patch_data(self, data, language_map): # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] - def __generate_property__(self, key, value): - result = { - key: { - "type": "Property", - "value": value - } - } - - return result - def __complete_label__(self, title, data): try: key = self.get_key(requested_key='rdfs:label') From a0cecf985fc9d1ddc214bcfe8803912f915cc32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 25 Jan 2023 18:17:58 +0100 Subject: [PATCH 19/74] Update file permissions --- doc/API.md | 0 doc/SDMX to JSON-LD.drawio | 0 doc/api.yaml | 0 images/logo.png | Bin pyproject.toml | 0 requirements.txt | 0 sdmx2jsonld/README.md | 0 sdmx2jsonld/__init__.py | 0 sdmx2jsonld/common/classprecedence.py | 0 sdmx2jsonld/common/commonclass.py | 0 sdmx2jsonld/common/config.py | 0 sdmx2jsonld/common/datatypeconversion.py | 0 sdmx2jsonld/common/listmanagement.py | 0 sdmx2jsonld/common/rdf.py | 0 sdmx2jsonld/common/regparser.py | 0 sdmx2jsonld/exceptions/__init__.py | 0 sdmx2jsonld/grammar/grammar.lark | 0 sdmx2jsonld/transform/__init__.py | 0 sdmx2jsonld/transform/attribute.py | 0 sdmx2jsonld/transform/catalogue.py | 0 sdmx2jsonld/transform/concept.py | 0 sdmx2jsonld/transform/conceptschema.py | 0 sdmx2jsonld/transform/context.py | 0 sdmx2jsonld/transform/dataset.py | 0 sdmx2jsonld/transform/dimension.py | 0 sdmx2jsonld/transform/entitytype.py | 0 sdmx2jsonld/transform/parser.py | 0 sdmx2jsonld/transform/property.py | 0 sdmx2jsonld/transform/transformer.py | 0 tests/__init__.py | 0 tests/common/__init__.py | 0 tests/common/test_classprecedence.py | 0 tests/docker-compose.yml | 0 33 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 doc/API.md mode change 100755 => 100644 doc/SDMX to JSON-LD.drawio mode change 100755 => 100644 doc/api.yaml mode change 100755 => 100644 images/logo.png mode change 100755 => 100644 pyproject.toml mode change 100755 => 100644 requirements.txt mode change 100755 => 100644 sdmx2jsonld/README.md mode change 100755 => 100644 sdmx2jsonld/__init__.py mode change 100755 => 100644 sdmx2jsonld/common/classprecedence.py mode change 100755 => 100644 sdmx2jsonld/common/commonclass.py mode change 100755 => 100644 sdmx2jsonld/common/config.py mode change 100755 => 100644 sdmx2jsonld/common/datatypeconversion.py mode change 100755 => 100644 sdmx2jsonld/common/listmanagement.py mode change 100755 => 100644 sdmx2jsonld/common/rdf.py mode change 100755 => 100644 sdmx2jsonld/common/regparser.py mode change 100755 => 100644 sdmx2jsonld/exceptions/__init__.py mode change 100755 => 100644 sdmx2jsonld/grammar/grammar.lark mode change 100755 => 100644 sdmx2jsonld/transform/__init__.py mode change 100755 => 100644 sdmx2jsonld/transform/attribute.py mode change 100755 => 100644 sdmx2jsonld/transform/catalogue.py mode change 100755 => 100644 sdmx2jsonld/transform/concept.py mode change 100755 => 100644 sdmx2jsonld/transform/conceptschema.py mode change 100755 => 100644 sdmx2jsonld/transform/context.py mode change 100755 => 100644 sdmx2jsonld/transform/dataset.py mode change 100755 => 100644 sdmx2jsonld/transform/dimension.py mode change 100755 => 100644 sdmx2jsonld/transform/entitytype.py mode change 100755 => 100644 sdmx2jsonld/transform/parser.py mode change 100755 => 100644 sdmx2jsonld/transform/property.py mode change 100755 => 100644 sdmx2jsonld/transform/transformer.py mode change 100755 => 100644 tests/__init__.py mode change 100755 => 100644 tests/common/__init__.py mode change 100755 => 100644 tests/common/test_classprecedence.py mode change 100755 => 100644 tests/docker-compose.yml diff --git a/doc/API.md b/doc/API.md old mode 100755 new mode 100644 diff --git a/doc/SDMX to JSON-LD.drawio b/doc/SDMX to JSON-LD.drawio old mode 100755 new mode 100644 diff --git a/doc/api.yaml b/doc/api.yaml old mode 100755 new mode 100644 diff --git a/images/logo.png b/images/logo.png old mode 100755 new mode 100644 diff --git a/pyproject.toml b/pyproject.toml old mode 100755 new mode 100644 diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/README.md b/sdmx2jsonld/README.md old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/__init__.py b/sdmx2jsonld/__init__.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/config.py b/sdmx2jsonld/common/config.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/rdf.py b/sdmx2jsonld/common/rdf.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/common/regparser.py b/sdmx2jsonld/common/regparser.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/exceptions/__init__.py b/sdmx2jsonld/exceptions/__init__.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/grammar/grammar.lark b/sdmx2jsonld/grammar/grammar.lark old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/__init__.py b/sdmx2jsonld/transform/__init__.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/attribute.py b/sdmx2jsonld/transform/attribute.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/context.py b/sdmx2jsonld/transform/context.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/dimension.py b/sdmx2jsonld/transform/dimension.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py old mode 100755 new mode 100644 diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py old mode 100755 new mode 100644 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100755 new mode 100644 diff --git a/tests/common/__init__.py b/tests/common/__init__.py old mode 100755 new mode 100644 diff --git a/tests/common/test_classprecedence.py b/tests/common/test_classprecedence.py old mode 100755 new mode 100644 diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml old mode 100755 new mode 100644 From 8eb3e2b4879f776f4fdebefe41e6e8dfc19d7d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 27 Jan 2023 18:23:16 +0100 Subject: [PATCH 20/74] Adding the list of dimensions into the Observation --- sdmx2jsonld/common/regparser.py | 12 +++++++---- sdmx2jsonld/transform/observation.py | 31 +++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/sdmx2jsonld/common/regparser.py b/sdmx2jsonld/common/regparser.py index 39e8061..1a2570d 100644 --- a/sdmx2jsonld/common/regparser.py +++ b/sdmx2jsonld/common/regparser.py @@ -25,24 +25,28 @@ class RegParser: def __init__(self): - regex = "http[s]?:\/\/(.*)" + regex = "http[s]?:\/\/(.+)" # Compile the Regex self.re = re.compile(regex) - def obtain_id(self, string_to_parse): + def obtain_id(self, string_to_parse, prefix_string=''): # Return if the string matched the ReGex out = self.re.match(string_to_parse) if out is None: # Check if the prefixed name include ':' - obtained_id = string_to_parse.split(':')[1] + try: + obtained_id = string_to_parse.split(':')[1] + except IndexError: + # We have a normal prefix or data + obtained_id = string_to_parse else: # We have a URIREF out = out.group(1) out = out.split("/") # we get the last value which corresponds to the id - obtained_id = out[(len(out) - 1):][0] + obtained_id = prefix_string + out[(len(out) - 1):][0] return obtained_id diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index 67c2de7..ab7cf3d 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -27,7 +27,7 @@ from sdmx2jsonld.sdmxattributes.code import Code from sdmx2jsonld.sdmxattributes.frequency import Frequency from sdmx2jsonld.common.regparser import RegParser -from re import search +from re import search, compile logger = getLogger() @@ -94,6 +94,7 @@ def __init__(self): self.concept_id = str() self.keys = {k: k for k in self.data.keys()} + self.regexpDimension = compile('ns1:.*') def add_data(self, title, observation_id, data): # We have a list of dimensions, a dataset, a list of attributes, a list of dimensions attributes @@ -143,6 +144,34 @@ def add_data(self, title, observation_id, data): identifier = parser.obtain_id(self.data[key]['object']) self.data[key]['object'] = 'urn:ngsi-ld:CatalogueDCAT-AP:' + identifier + # Add dimensions + result = self.__assign_dimensions__(data=data) + self.data['dimensions']['value'] = result + + def __assign_dimensions__(self, data): + def create_data(key, value): + new_dimension = { + "key": key, + "value": value + } + + return new_dimension + parser = RegParser() + + # We need to get the list of available dimension keys + dimension_keys = [a for a in data if self.regexpDimension.match(a.__str__()) is not None] + + # We need to get the list of values for these keys, if there is an url, it is considered a + values = [self.get_data(data[data.index(a) + 1]) for a in dimension_keys] + values = [parser.obtain_id(a, prefix_string="urn:ngsi-ld:Concept:") for a in values] + + # Get the list of Entity IDs + entity_ids = ["urn:ngsi-ld:DimensionProperty:" + b.split("ns1:", 1)[1] for b in dimension_keys] + + result = [create_data(key=entity_ids[i], value=values[i]) for i in range(0, len(values))] + + return result + def __assign_property__(self, requested_key, data, key_property='value'): key = self.get_key(requested_key=requested_key) position = data.index(requested_key) + 1 From e6a6b331f8ea27242ca126ac61f2445d32cb7979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 7 Feb 2023 19:15:14 +0100 Subject: [PATCH 21/74] Homogeneize all properties as normalized JSON-LD representation --- sdmx2jsonld/transform/conceptschema.py | 2 +- sdmx2jsonld/transform/dataset.py | 8 ++++---- sdmx2jsonld/transform/property.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 8388259..f7e7da0 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -39,7 +39,7 @@ def __init__(self): "value": list() }, "skos:hasTopConcept": { - "type": "Property", + "type": "Relationship", "value": list() }, diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 5a0a1c8..e1d56fd 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -67,7 +67,7 @@ def __init__(self): 'key': 'stat:attribute', 'value': { "stat:attribute": { - "type": "Property", + "type": "Relationship", "value": list() } } @@ -77,7 +77,7 @@ def __init__(self): 'key': 'stat:dimension', 'value': { "stat:dimension": { - "type": "Property", + "type": "Relationship", "value": list() } } @@ -87,7 +87,7 @@ def __init__(self): 'key': 'stat:statUnitMeasure', 'value': { "stat:statUnitMeasure": { - "type": "Property", + "type": "Relationship", "value": list() } } @@ -170,7 +170,7 @@ def patch_data(self, data, language_map): else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [self.data.update({k: v}) for k, v in data.items()] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] def __complete_label__(self, title, data): try: diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index 5682f44..b966036 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -60,7 +60,7 @@ def __init__(self, entity): "object": str() }, "qb:concept": { - "type": "Property", + "type": "Relationship", "value": str() }, "@context": dict() @@ -141,7 +141,7 @@ def add_data(self, id, data): ]) # add the new data to the dataset structure - [self.data.update({k: v}) for k, v in data.items()] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] # Simplify Context and order keys a = Context() From 0fc8bbba7b12d6505ea7549572e76f78f144c9d3 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Mon, 13 Mar 2023 13:15:24 +0100 Subject: [PATCH 22/74] Corrected boolean tests and requirements.txt --- requirements.txt | 3 ++- sdmx2jsonld/common/datatypeconversion.py | 4 +++- sdmx2jsonld/transform/catalogue.py | 8 ++++---- sdmx2jsonld/transform/concept.py | 7 +++++-- tests/common/test_datatypeconversion.py | 9 +++------ tests/kex/e1.ttl | 8 -------- 6 files changed, 17 insertions(+), 22 deletions(-) delete mode 100644 tests/kex/e1.ttl diff --git a/requirements.txt b/requirements.txt index 9e5b0e2..b1a3a83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ python-multipart==0.0.5 loguru==0.6.0 requests==2.28.1 rdflib~=6.2.0 - +dateutil>=2.8.2 +pytz>=2022.6 diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index cff1b16..ab8b0b4 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -40,7 +40,6 @@ def __init__(self): self.regex_false_date2 = compile(r"^%Y-%d-%m(.*)%f") def correct_datatype_format(self, format_dt: str, hour24: bool = True): - if hour24: format_dt = sub(self.regex_12hour, r"\1H\3", format_dt) @@ -54,6 +53,9 @@ def correct_datatype_format(self, format_dt: str, hour24: bool = True): def convert(self, data, datatype): def stoutc(value): + """ + Converts a date in string format to UTC date using + """ dt = parser.parse(value, tzinfos=whois_timezone_info) dt = dt.astimezone(pytz.UTC) return dt.replace(tzinfo=timezone.utc).isoformat() diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7502ee0..bdf24b7 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -35,8 +35,8 @@ def __init__(self): "id": str(), "type": "CatalogueDCAT-AP", "dataset": { - "type": "object", - "value": str() + "type": "Relationship", + "object": str() }, "language": { @@ -65,7 +65,7 @@ def __init__(self): }, "title": { - "type": "Array", + "type": "Property", "value": list() }, @@ -89,7 +89,7 @@ def add_dataset(self, dataset_id): self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + hash1 # Add dataset id - self.data['dataset']['value'] = dataset_id + self.data['dataset']['object'] = dataset_id def get(self): return self.data diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index db3b126..951e9ee 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -41,7 +41,7 @@ def __init__(self): }, "skos:inScheme": { "type": "Relationship", - "value": str() + "object": str() }, "rdfs:subClassOf": { "type": "Property", @@ -170,7 +170,10 @@ def need_add_in_scheme(self, data): parser = RegParser() concept_schema = data[position][0] concept_schema = "urn:ngsi-ld:ConceptSchema:" + parser.obtain_id(concept_schema) - self.data['skos:inScheme']['value'] = concept_schema + if self.data['skos:inScheme']['type'] == 'Relationship': + self.data['skos:inScheme']['object'] = concept_schema + else: + self.data['skos:inScheme']['value'] = concept_schema def need_add_notation(self, data): try: diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index 3d3d1b1..bc5c83c 100644 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -35,15 +35,12 @@ def test_string_to_bool(self): values = ("True", "true", "y", "yes", "T", "1", 1, True) for value in values: - assert (dtc.convert(value, token_type)) - - values = ("True", "true", "y", "yes", "T", "1", 1, True) - for value in values: - assert (dtc.convert(value, token_type)) + print(f"2 convert {value}") + assert (dtc.convert(f'"{value}"', token_type)) values = ("fAlsE", "False", "N", "No", "F", "0", "0.0", "", "None", None, [], {}, 0, 0.0) for value in values: - assert (not dtc.convert(value, token_type)) + assert (not dtc.convert(f'"{value}"', token_type)) invalid_values = (5, 4.2, "invalid value", "nil") for value in invalid_values: diff --git a/tests/kex/e1.ttl b/tests/kex/e1.ttl deleted file mode 100644 index 7a489a3..0000000 --- a/tests/kex/e1.ttl +++ /dev/null @@ -1,8 +0,0 @@ -@prefix rdf: . -@prefix contact: . - - - rdf:type contact:Person; - contact:fullName "Eric Miller"; - contact:mailbox ; - contact:personalTitle "Dr.". From ecb0c51d7b81545c95053962b6490526a87c1786 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 22 Mar 2023 11:58:17 +0100 Subject: [PATCH 23/74] Added some tests for common and slightly changed commonclass.py --- .vscode/settings.json | 11 ++++++++++ requirements.txt | 2 +- sdmx2jsonld/common/commonclass.py | 1 + sdmx2jsonld/common/rdf.py | 4 ++++ tests/common/test_commonclass.py | 12 ++++++++++ tests/common/test_datatypeconversion.py | 1 - tests/common/test_rdf.py | 29 +++++++++++++++++++++++++ tests/common/test_regparser.py | 11 ++++++++++ 8 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 tests/common/test_rdf.py create mode 100644 tests/common/test_regparser.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e9e6a80 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.testing.unittestArgs": [ + "-v", + "-s", + "./tests", + "-p", + "test_*.py" + ], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b1a3a83..43467fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ python-multipart==0.0.5 loguru==0.6.0 requests==2.28.1 rdflib~=6.2.0 -dateutil>=2.8.2 +python-dateutil>=2.8.2 pytz>=2022.6 diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index f58b66d..5d135ba 100644 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -85,4 +85,5 @@ def generate_id(self, value, entity=None): else: aux = "urn:ngsi-ld:" + entity + ":" + aux + self.data['id'] = aux return aux diff --git a/sdmx2jsonld/common/rdf.py b/sdmx2jsonld/common/rdf.py index 9e7808b..1eff170 100644 --- a/sdmx2jsonld/common/rdf.py +++ b/sdmx2jsonld/common/rdf.py @@ -23,6 +23,10 @@ def turtle_terse(rdf_content): + ''' + Function which receives a string in formatted with RDF, dialect turtle, parses it and returns the same data + parsed in another string. The incoming data in the parameter is equivalent to the returned data. + ''' # Create a Graph g2 = Graph() diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py index 558aae7..1b5e8a5 100644 --- a/tests/common/test_commonclass.py +++ b/tests/common/test_commonclass.py @@ -23,6 +23,7 @@ from unittest import TestCase from sdmx2jsonld.common.commonclass import CommonClass import os +import json class TestCommonClass(TestCase): def setUp(self) -> None: @@ -36,10 +37,21 @@ def test_instance_class(self): # print(urnid) def test_save(self): + context = {"@context": "https://raw.githubusercontent.com/smart-data-models/data-models/master/context/merge_subjects_config_example.json"} + context_map = {"address": "https://smartdatamodels.org/address", + "alternateName": "https://smartdatamodels.org/alternateName", + "status": "ngsi-ld:status"} cclass = CommonClass("test.common.entity") urnid = cclass.generate_id("https://string-to-parse-ur/entity_id") + assert(urnid == "urn:ngsi-ld:test.common.entity:entity_id") + + cclass.add_context(context, context_map) os.makedirs("/tmp/commonclass", exist_ok=True) os.chdir("/tmp/commonclass") cclass.save() + with open("/tmp/commonclass/output/test.common.entity_entity_id.jsonld", "r") as f: + data = json.load(f) + assert(data['id'] == urnid) + assert(data['@context'] == context['@context']) diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index bc5c83c..1462de0 100644 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -3,7 +3,6 @@ import sys - class TestDataTypeConversion(TestCase): def setUp(self) -> None: pass diff --git a/tests/common/test_rdf.py b/tests/common/test_rdf.py new file mode 100644 index 0000000..507cb06 --- /dev/null +++ b/tests/common/test_rdf.py @@ -0,0 +1,29 @@ +from unittest import TestCase +from sdmx2jsonld.common.rdf import turtle_terse +from rdflib import Graph + +class TestRegToParser(TestCase): + def setUp(self) -> None: + pass + + def test_turtle_1(self): + rdf_data = ''' +@prefix ab: . + +ab:richard ab:homeTel "(229) 276-5135" . +ab:richard ab:email "richard49@hotmail.com" . + +ab:cindy ab:homeTel "(245) 646-5488" . +ab:cindy ab:email "cindym@gmail.com" . + +ab:craig ab:homeTel "(194) 966-1505" . +ab:craig ab:email "craigellis@yahoo.com" . +ab:craig ab:email "c.ellis@usairwaysgroup.com" . + ''' + rdf_content = turtle_terse(rdf_data) + + gx = Graph() + gx = gx.parse(data=rdf_content, format="turtle") + ser = gx.serialize(format="turtle") + + assert(rdf_content == ser) diff --git a/tests/common/test_regparser.py b/tests/common/test_regparser.py new file mode 100644 index 0000000..851ef94 --- /dev/null +++ b/tests/common/test_regparser.py @@ -0,0 +1,11 @@ +from unittest import TestCase +from sdmx2jsonld.common.regparser import RegParser +class TestRegToParser(TestCase): + def setUp(self) -> None: + pass + + def test_get_id(self): + re = RegParser() + assert(re.obtain_id("https://elmundo.es/episode-one") == "episode-one") + + From ea83040a0ce02b8d504bdb2c087115c61cdb915a Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 5 Apr 2023 09:48:01 +0200 Subject: [PATCH 24/74] Added another test for lists --- sdmx2jsonld/common/listmanagement.py | 15 ++++++++++++++- tests/common/test_listmanagement.py | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/common/test_listmanagement.py diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index eeae59f..fe4b2a5 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -38,7 +38,7 @@ def filter_key_with_prefix(prefix_key, not_allowed_keys, further_process_keys): else: if aux[1] not in ['component', 'label']: # These are the identified not allowed keys, we need to inform about them - logger.warn(f'The property {aux[1]} is not supported in statDCAT-AP') + logger.warning(f'The property {aux[1]} is not supported in statDCAT-AP') else: # These are the identified keys managed in a different way logger.info(f'The property {aux[1]} is manage afterwards in Dataset Class or in Property Class') @@ -56,6 +56,14 @@ def flatten_value(y): def get_rest_data(data, not_allowed_keys, further_process_keys): + with open("/tmp/output.txt", "a") as f: + f.write("---- data ----- \n") + f.write(str(data)) + f.write("\n---- not_allowed_keys ----- \n") + f.write(str(not_allowed_keys)) + f.write("\n---- not_allowed_keys ----- \n") + f.write(str(further_process_keys)) + aux = {data[i]: flatten_value(data[i + 1]) for i in range(0, len(data), 2)} # We need to get the list of keys from the dict @@ -64,4 +72,9 @@ def get_rest_data(data, not_allowed_keys, further_process_keys): new_data = {k: aux[k] for k in new_keys} + + with open("/tmp/output.txt", "a") as f: + f.write("\n\n---- OUTPUT ----- \n") + f.write(str(new_data)) + f.write("\n---- END OUTPUT ----- \n\n") return new_data diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py new file mode 100644 index 0000000..9da3a99 --- /dev/null +++ b/tests/common/test_listmanagement.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value + +class TestRegToParser(TestCase): + def setUp(self) -> None: + pass + + def test_get_rest_data(self): + data = ['a', ['qb:AttributeProperty'], 'rdfs:label', [['"SDMX attribute COMMENT_OBS"', '@en'], ['"Attribut SDMX "', '@fr']], 'dct:created', [['2022-01-15T06:00:00+00:00']], 'dct:identifier', [['"a3003"']], 'dct:modified', [['2022-01-15T06:30:00+00:00']], 'qb:concept', ['http://bauhaus/concepts/definition/c4303'], 'insee:disseminationStatus', ['http://id.insee.fr/codes/base/statutDiffusion/Prive'], 'insee:validationState', [['"Unpublished"']], 'rdfs:range', ['xsd:string'], 'skos:notation', [['"COMMENT_OBS"']]] + not_allowed_keys = ['sliceKey', 'component', 'disseminationStatus', 'validationState', 'notation', 'label', 'codeList', 'concept'] + further_process_keys = ['component', 'label'] + expected_res = {'dct:created': '2022-01-15T06:00:00+00:00', 'dct:identifier': 'a3003', 'dct:modified': '2022-01-15T06:30:00+00:00', 'rdfs:range': 'xsd:string'} + res = get_rest_data(data, not_allowed_keys, further_process_keys) + assert(expected_res == res) + + def test_flatten_value(self): + data = [['"SDMX attribute PRE_BREAK_VALUE"', '@en'], ['"Attribut SDMX "', '@fr']] + expected_res = 'SDMX attribute PRE_BREAK_VALUE' + assert(flatten_value(data) == expected_res) + + data = [["'another thing'", 48],["another one array"], 54] + expected_res = "'another thing'" + assert(flatten_value(data) == expected_res) + From 5dd1379414626d7dd4f898dea41b854b0a1be63b Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 6 Apr 2023 13:25:27 +0200 Subject: [PATCH 25/74] solved CB writings --- ngsild/ngsild_connector.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index ab07113..c0e0082 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -15,23 +15,21 @@ def __init__(self, path=None): self.base_url = config['broker'] def get_url(self): - url = f"{self.base_url}/ngsi-ld/v1/entityOperations/create" + url = f"{self.base_url}/ngsi-ld/v1/entities" return url - def send_pretty_much_data(self, json_object): + def send_data_array(self, json_object): d = json.loads(json_object) d = d if type(d) is list else [d] for elem in d: rc, r = c.send_data(json.dumps(elem)) - print("Code: ", rc) - print(r) def send_data(self, json_object): # Send the data to a FIWARE Context Broker instance headers = { - 'Content-Type': 'application/ld+json', - 'Accept': 'application/ld+json' + 'Content-Type': 'application/ld+json' + #, 'Accept': 'application/ld+json' } url = self.get_url() @@ -39,9 +37,12 @@ def send_data(self, json_object): r = post(url=url, headers=headers, data=json_object, timeout=5) - resp = json.loads(r.text) + # resp = json.loads(r.text) response_status_code = r.status_code + if response_status_code == 201: + print("LOCATION: ", r.headers['Location']) + # Let exceptions raise.... They can be controlled somewhere else. return response_status_code, resp @@ -58,12 +59,4 @@ def send_data(self, json_object): rdf_data = rf.read() r = parser.parsing(StringIO(rdf_data), out=False) - - with open("/tmp/ww", "w") as f: - f.write(r) - - # print(data) - c.send_pretty_much_data(r) - # rc, r = c.send_data(data) - # print("Code: ", rc) - # print(r) + c.send_data_array(r) From b446d01c00af70e77b45bf00012bf5ce51ff04e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 24 Apr 2023 13:49:59 +0200 Subject: [PATCH 26/74] Resolution of several bugs --- sdmx2jsonld/common/listmanagement.py | 13 ++++++++++++- sdmx2jsonld/transform/catalogue.py | 6 +++--- sdmx2jsonld/transform/conceptschema.py | 4 ++-- sdmx2jsonld/transform/dataset.py | 10 +++++----- sdmx2jsonld/transform/parser.py | 18 ++++++++++++------ sdmx2jsonld/transform/property.py | 4 ++-- sdmx2jsonld/transform/transformer.py | 13 ++++++++----- 7 files changed, 44 insertions(+), 24 deletions(-) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index 131f200..b738256 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -50,7 +50,18 @@ def filter_key_with_prefix(prefix_key, not_allowed_keys, further_process_keys): def flatten_value(y): if isinstance(y, list): - return flatten_value(y[0]) + aux = len(y) + if aux == 1: + return flatten_value(y[0]) + elif aux > 1: + # for each element of the list we have to flatten to string and create the corresponding list + # this case corresponds to multilingual content + result = dict() + for i in range(0, aux): + result[y[i][1][1:]] = flatten_value(y[i][0]) + return result + else: # in case of len == 0 be return the empty string + return '' else: return y.replace('"', '') diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index c7c1874..f3d43b0 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -37,7 +37,7 @@ def __init__(self): "type": "CatalogueDCAT-AP", "qb:dataset": { "type": "Relationship", - "value": str() + "object": str() }, "dct:language": { @@ -85,7 +85,7 @@ def add_dataset(self, dataset_id): self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + hash1 # Add dataset id - self.data['qb:dataset']['value'] = dataset_id + self.data['qb:dataset']['object'] = dataset_id def add_data(self, title, dataset_id, data): # We need to complete the data corresponding to the Catalogue: rdfs:label @@ -106,7 +106,7 @@ def add_data(self, title, dataset_id, data): # Add structure key = self.get_key(requested_key='qb:structure') position = data.index(key) + 1 - self.data['qb:dataset']['value'] = data[position][0] + self.data['qb:dataset']['object'] = data[position][0] # Get the rest of the data data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher']) diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index f7e7da0..8579792 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -40,7 +40,7 @@ def __init__(self): }, "skos:hasTopConcept": { "type": "Relationship", - "value": list() + "object": list() }, @@ -107,7 +107,7 @@ def add_data(self, concept_schema_id, data): # skos:hasTopConcept, this is a list of ids position = data.index('skos:hasTopConcept') + 1 result = list(map(lambda x: self.generate_id(value=x, entity='Concept'), data[position])) - self.data['skos:hasTopConcept']['value'] = result + self.data['skos:hasTopConcept']['object'] = result # Simplify Context and order keys a = Context() diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index e1d56fd..8cb0aa8 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -68,7 +68,7 @@ def __init__(self): 'value': { "stat:attribute": { "type": "Relationship", - "value": list() + "object": list() } } }, @@ -78,7 +78,7 @@ def __init__(self): 'value': { "stat:dimension": { "type": "Relationship", - "value": list() + "object": list() } } }, @@ -88,7 +88,7 @@ def __init__(self): 'value': { "stat:statUnitMeasure": { "type": "Relationship", - "value": list() + "object": list() } } } @@ -112,11 +112,11 @@ def add_components(self, context, component): key = self.components[type_component]['key'] # It is possible that the original file contains already the description - if new_id in self.components[type_component]['value'][key]['value']: + if new_id in self.components[type_component]['value'][key]['object']: logger.warning( f"The component {new_id} is duplicated and already defined in the {self.data['id']}") else: - self.components[type_component]['value'][key]['value'].append(new_id) + self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] except ValueError: logger.error(f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}") diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 591921b..f60397f 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -84,13 +84,14 @@ def parsing_file(self, content: TextIOWrapper, out: bool): transform.save() elif content is not None: print() + pprint(transform.get_catalogue()) - pprint(transform.get_observation()) - pprint(transform.get_dataset()) + self.__check_pprint__(transform.get_observation()) + self.__check_pprint__(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] - [pprint(x.get()) for x in transform.get_conceptSchemas()] - [pprint(x.get()) for x in transform.get_conceptLists()] + [pprint(x.get()) for x in transform.get_concept_schemas()] + [pprint(x.get()) for x in transform.get_concept_lists()] def parsing_string(self, content: StringIO): transform = TreeToJson() @@ -108,8 +109,8 @@ def parsing_string(self, content: StringIO): result.append(transform.get_dataset()) [result.append(x.get()) for x in transform.get_dimensions()] [result.append(x.get()) for x in transform.get_attributes()] - [result.append(x.get()) for x in transform.get_conceptSchemas()] - [result.append(x.get()) for x in transform.get_conceptLists()] + [result.append(x.get()) for x in transform.get_concept_schemas()] + [result.append(x.get()) for x in transform.get_concept_lists()] json_object = dumps(result, indent=4, ensure_ascii=False) @@ -117,3 +118,8 @@ def parsing_string(self, content: StringIO): outfile.write(json_object) return json_object + + @staticmethod + def __check_pprint__(data): + if data is not None: + pprint(data) diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index b966036..d515379 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -61,7 +61,7 @@ def __init__(self, entity): }, "qb:concept": { "type": "Relationship", - "value": str() + "object": str() }, "@context": dict() } @@ -121,7 +121,7 @@ def add_data(self, id, data): # TODO: the concept id need to check if it is a normal id or an url position = data.index('qb:concept') + 1 concept = self.generate_id(entity="Concept", value=data[position][0]) - self.data['qb:concept']['value'] = concept + self.data['qb:concept']['object'] = concept # Get the rest of the data data = get_rest_data(data=data, diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index a1062a2..0fed610 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -116,10 +116,12 @@ def get_catalogue(self): return self.entity_type.get_catalogue() def get_observation(self): - return self.entity_type.get_observation() + if self.entity_type.observation.data['id'] != '': + return self.entity_type.get_observation() def get_dataset(self): - return self.entity_type.get_dataset() + if self.entity_type.dataset.data['id'] != '': + return self.entity_type.get_dataset() def get_dimensions(self): return self.entity_type.get_dimensions() @@ -127,16 +129,17 @@ def get_dimensions(self): def get_attributes(self): return self.entity_type.get_attributes() - def get_conceptSchemas(self): + def get_concept_schemas(self): return self.entity_type.get_conceptSchemas() - def get_conceptLists(self): + def get_concept_lists(self): return self.entity_type.get_conceptList() def save(self): self.entity_type.save('catalogue') - self.entity_type.save('observation') + if self.entity_type.observation.data['id'] != '': + self.entity_type.save('observation') if self.entity_type.dataset.data['id'] != '': self.entity_type.save('dataset') From cacd7a11e67b45331583744779413ebef25f3cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 25 Apr 2023 15:43:22 +0200 Subject: [PATCH 27/74] Resolution some bugs --- sdmx2jsonld/common/listmanagement.py | 14 ++++++++--- sdmx2jsonld/transform/attribute.py | 2 +- sdmx2jsonld/transform/catalogue.py | 32 +++++++++++++------------- sdmx2jsonld/transform/conceptschema.py | 14 +++++++++++ sdmx2jsonld/transform/dimension.py | 6 ++--- sdmx2jsonld/transform/entitytype.py | 2 +- sdmx2jsonld/transform/property.py | 6 ++--- 7 files changed, 49 insertions(+), 27 deletions(-) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index b738256..b50feea 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -55,10 +55,18 @@ def flatten_value(y): return flatten_value(y[0]) elif aux > 1: # for each element of the list we have to flatten to string and create the corresponding list + # We need to differentiate between multilingual case and multiple data # this case corresponds to multilingual content - result = dict() - for i in range(0, aux): - result[y[i][1][1:]] = flatten_value(y[i][0]) + if len(y[0]) == 2: + # Multilingual case + result = dict() + for i in range(0, aux): + result[y[i][1][1:]] = flatten_value(y[i][0]) + else: + # Normal case + result = list() + for i in range(0, aux): + result.append(flatten_value(y[i])) return result else: # in case of len == 0 be return the empty string return '' diff --git a/sdmx2jsonld/transform/attribute.py b/sdmx2jsonld/transform/attribute.py index f5fc523..612137e 100644 --- a/sdmx2jsonld/transform/attribute.py +++ b/sdmx2jsonld/transform/attribute.py @@ -29,7 +29,7 @@ def __init__(self): self.data['type'] = 'AttributeProperty' def add_data(self, attribute_id, data): - super().add_data(id=id, data=data) + super().add_data(property_id=attribute_id, data=data) # Add the id self.data['id'] = "urn:ngsi-ld:AttributeProperty:" + attribute_id diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index f3d43b0..8c9a820 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -23,6 +23,7 @@ from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.common.listmanagement import get_rest_data +from sdmx2jsonld.transform.context import Context import random logger = getLogger() @@ -68,7 +69,13 @@ def __init__(self): "dct:title": { "type": "Property", "value": list() - } + }, + + "@context": [ + "https://raw.githubusercontent.com/SEMICeu/DCAT-AP/master/releases/1.1/dcat-ap_1.1.jsonld", + "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" + ] + } self.concept_id = str() @@ -103,25 +110,18 @@ def add_data(self, title, dataset_id, data): position = data.index(key) + 1 self.data['dct:publisher']['value'] = data[position][0] - # Add structure - key = self.get_key(requested_key='qb:structure') - position = data.index(key) + 1 - self.data['qb:dataset']['object'] = data[position][0] - - # Get the rest of the data - data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher']) + # Get the rest of the data, qb:structure has the same value of qb:dataset so we decide to + # use only qb:dataset in CatalogueDCAT-AP + data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher', 'structure']) # add the new data to the dataset structure self.patch_data(data, False) - # Add context - context = { - "@context": [ - "https://raw.githubusercontent.com/SEMICeu/DCAT-AP/master/releases/1.1/dcat-ap_1.1.jsonld", - "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" - ] - } - self.data.update(context) + # Order Context keys + a = Context() + a.set_data(data=self.data) + a.order_context() + self.data = a.get_data() def patch_data(self, data, language_map): if language_map: diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 8579792..e90edcd 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -23,6 +23,7 @@ from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.transform.context import Context +from sdmx2jsonld.common.listmanagement import flatten_value logger = getLogger() @@ -109,6 +110,19 @@ def add_data(self, concept_schema_id, data): result = list(map(lambda x: self.generate_id(value=x, entity='Concept'), data[position])) self.data['skos:hasTopConcept']['object'] = result + # Get the rest of data, dct:created and dct:modified properties + position = data.index('dct:created') + 1 + self.data['dct:created'] = { + "type": "Property", + "value": flatten_value(data[position]) + } + + position = data.index('dct:modified') + 1 + self.data['dct:modified'] = { + "type": "Property", + "value": flatten_value(data[position]) + } + # Simplify Context and order keys a = Context() a.set_data(data=self.data) diff --git a/sdmx2jsonld/transform/dimension.py b/sdmx2jsonld/transform/dimension.py index bbbd68c..de03424 100644 --- a/sdmx2jsonld/transform/dimension.py +++ b/sdmx2jsonld/transform/dimension.py @@ -28,8 +28,8 @@ def __init__(self): super().__init__(entity='DimensionProperty') self.data['type'] = 'DimensionProperty' - def add_data(self, id, data): - super().add_data(id=id, data=data) + def add_data(self, property_id, data): + super().add_data(property_id=property_id, data=data) # Add the id - self.data['id'] = "urn:ngsi-ld:DimensionProperty:" + id + self.data['id'] = "urn:ngsi-ld:DimensionProperty:" + property_id diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 225097d..efaed04 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -167,7 +167,7 @@ def create_data(self, type, data, title): dimension = Dimension() dimension.add_context(context=self.context, context_mapping=self.context_mapping) dimension_id = parser.obtain_id(title) - dimension.add_data(id=dimension_id, data=data) + dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) elif type == 'Attribute': attribute = Attribute() diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index d515379..94e8802 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -68,7 +68,7 @@ def __init__(self, entity): self.keys = {k: k for k in self.data.keys()} - def add_data(self, id, data): + def add_data(self, property_id, data): # TODO: We have to control that data include the indexes that we want to search # We need to complete the data corresponding to the Dimension: rdfs:label position = data.index('rdfs:label') + 1 @@ -80,7 +80,7 @@ def add_data(self, id, data): try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The Property {id} has a ' + logger.warning(f'The Property {property_id} has a ' f'rdfs:label without language tag: {description}') aux = len(description) @@ -112,7 +112,7 @@ def add_data(self, id, data): code_list = self.generate_id(entity="ConceptSchema", value=data[position][0]) self.data['qb:codeList']['object'] = code_list except ValueError: - logger.warning(f'Property: {id} has not qb:codeList, deleting the key in the data') + logger.warning(f'Property: {property_id} has not qb:codeList, deleting the key in the data') # If we have not the property, we delete it from data self.data.pop('qb:codeList') From 60a7460b53fd28c87771058c3b84d7692850fd31 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 16 May 2023 11:29:48 +0200 Subject: [PATCH 28/74] Repaired broken tests after last merge --- ngsild/ngsild_connector.py | 1 - tests/common/test_datatypeconversion.py | 4 +--- tests/common/test_listmanagement.py | 9 +++------ tests/test1.py | 19 ------------------- 4 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 tests/test1.py diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index c0e0082..06cd213 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -2,7 +2,6 @@ import json from requests import post, exceptions - class NGSILDConnector: def __init__(self, path=None): if path == None: diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index 48a23c1..8f13e9b 100755 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -55,7 +55,6 @@ def test_string_to_bool(self): values = ("True", "true", "y", "yes", "T", "1", 1, True) for value in values: - print(f"2 convert {value}") assert (dtc.convert(f'"{value}"', token_type)) values = ("fAlsE", "False", "N", "No", "F", "0", "0.0", "", "None", None, [], {}, 0, 0.0) @@ -88,7 +87,6 @@ def test_string_to_dates(self): d = zip(dates, expected) for test_date, expected_date in d: - print(test_date, " | ", dtc.convert(test_date, token_type)) assert (expected_date == dtc.convert(test_date, token_type)) class Test(TestCase): @@ -99,7 +97,7 @@ def test_datetime_string_conversion_1(self): """ Check if we can get a correct datetime value from a string, case 1 """ - obtained = self.conversion.convert('"2022-01-15T08:00:00.000"', 'xsd:dateTime') + obtained = self.conversion.convert('"2022-01-15T08:00:00.000 UTC"', 'xsd:dateTime') expected = '2022-01-15T08:00:00+00:00' assert obtained == expected, f"\n\nDateTime was not the expected," \ f"\n got : {obtained}" \ diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index 9da3a99..d44a7f3 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -15,10 +15,7 @@ def test_get_rest_data(self): def test_flatten_value(self): data = [['"SDMX attribute PRE_BREAK_VALUE"', '@en'], ['"Attribut SDMX "', '@fr']] - expected_res = 'SDMX attribute PRE_BREAK_VALUE' - assert(flatten_value(data) == expected_res) - - data = [["'another thing'", 48],["another one array"], 54] - expected_res = "'another thing'" - assert(flatten_value(data) == expected_res) + expected_res = {'en': 'SDMX attribute PRE_BREAK_VALUE', 'fr': 'Attribut SDMX '} + got_data = flatten_value(data) + assert(got_data == expected_res) diff --git a/tests/test1.py b/tests/test1.py deleted file mode 100644 index 288d4a9..0000000 --- a/tests/test1.py +++ /dev/null @@ -1,19 +0,0 @@ -from sdmx2jsonld.transform.parser import Parser -from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken - -if __name__ == "__main__": - # file_in = open("../examples/structures-accounts.ttl") - file_in = open("kex/e1.ttl") - generate_files = True - - # Start parsing the file - my_parser = Parser() - - try: - my_parser.parsing(content=file_in, out=generate_files) - except UnexpectedToken as e: - print(e) - except UnexpectedInput as e: - print(e) - except UnexpectedEOF as e: - print(e) From 6ece50fae7c3d7e21a250e80c29a78f7ded3a10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 14 Jun 2023 10:10:39 +0200 Subject: [PATCH 29/74] Update requirements file --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8d9429..f93535b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ secure==0.3.0 docopt==0.6.2 schema==0.7.5 hi-dateinfer==0.4.6 -fastapi==0.88.0 -uvicorn==0.20.0 -python-multipart==0.0.5 -loguru==0.6.0 -requests==2.28.1 -rdflib~=6.2.0 +fastapi==0.95.1 +uvicorn==0.22.0 +python-multipart==0.0.6 +loguru==0.7.0 +requests==2.31.0 +rdflib>=6.2,<6.4 python-dateutil>=2.8.2 pytz>=2022.6 From a5752f2cc24a0289f5f4cd5434478f2f02485949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 15 Jun 2023 14:33:19 +0200 Subject: [PATCH 30/74] Remove VSCode configuration file --- .gitignore | 1 + .vscode/settings.json | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index eacca3c..cedcb13 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,7 @@ dmypy.json # IDE .idea +.vscode # Temporal generated JSON-LD documents *.jsonld diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e9e6a80..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "python.testing.unittestArgs": [ - "-v", - "-s", - "./tests", - "-p", - "test_*.py" - ], - "python.testing.pytestEnabled": false, - "python.testing.unittestEnabled": true -} \ No newline at end of file From fa366dc32d1f2966eecce9a8f72af1f5b7e8590d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 15 Jun 2023 17:04:11 +0200 Subject: [PATCH 31/74] Resolving sdmx-dimensions issues --- requirements.txt | 2 +- sdmx2jsonld/common/commonclass.py | 13 ++++++++----- sdmx2jsonld/common/listmanagement.py | 6 +++++- sdmx2jsonld/transform/dataset.py | 12 +++++++++++- sdmx2jsonld/transform/entitytype.py | 23 +++++++++-------------- 5 files changed, 34 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index f93535b..363db7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ secure==0.3.0 docopt==0.6.2 schema==0.7.5 hi-dateinfer==0.4.6 -fastapi==0.95.1 +fastapi==0.97.0 uvicorn==0.22.0 python-multipart==0.0.6 loguru==0.7.0 diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index 6008da6..1e80c7b 100644 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -76,17 +76,20 @@ def save(self): with open(filename, "w") as outfile: outfile.write(json_object) - def generate_id(self, value, entity=None): + def generate_id(self, value, entity=None, update_id=False): parse = RegParser() aux = parse.obtain_id(value) if entity is None: - aux = "urn:ngsi-ld:" + self.entity + ":" + aux + new_aux = "urn:ngsi-ld:" + self.entity + ":" + aux else: - aux = "urn:ngsi-ld:" + entity + ":" + aux + new_aux = "urn:ngsi-ld:" + entity + ":" + aux - self.data['id'] = aux - return aux + if update_id: + self.data['id'] = new_aux + return new_aux + else: + return aux, new_aux def __generate_property__(self, key, value): result = { diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index 5dea1fc..40be8b7 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -74,7 +74,11 @@ def flatten_value(y): return y.replace('"', '') -def get_rest_data(data, not_allowed_keys=[], further_process_keys=[]): +def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): + if further_process_keys is None: + further_process_keys = [] + if not_allowed_keys is None: + not_allowed_keys = [] aux = {data[i]: flatten_value(data[i + 1]) for i in range(0, len(data), 2)} # We need to get the list of keys from the dict diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 8cb0aa8..02b2467 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -103,18 +103,28 @@ def add_components(self, context, component): # We need to know which kind of component we have, it should be the verb: # qb:attribute, qb:dimension, or qb:measure list_components = ['qb:attribute', 'qb:dimension', 'qb:measure'] + + # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment + # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix + list_special_dimensions = ['freq', 'refArea', 'timePeriod'] + type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 try: entity = self.components[type_component]['entity'] - new_id = self.generate_id(entity=entity, value=component[position][0]) + name, new_id = self.generate_id(entity=entity, value=component[position][0], update_id=False) key = self.components[type_component]['key'] # It is possible that the original file contains already the description if new_id in self.components[type_component]['value'][key]['object']: logger.warning( f"The component {new_id} is duplicated and already defined in the {self.data['id']}") + elif name in list_special_dimensions: + # We need to create manually the description of these dimensions + logger.warning( + f"The component {name} is defined probably outside of the file, " + f"creating manually the DimensionsProperty entity") else: self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index efaed04..79a1f81 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -63,6 +63,7 @@ def __init__(self): self.observation = Observation() self.pre = Precedence() + self.parser = RegParser() def __find_entity_type__(self, string): """ @@ -142,22 +143,16 @@ def flatten_value(y): self.dataset.patch_data(data=flatten_data, language_map=language_map) def create_data(self, type, data, title): - parser = RegParser() - if type == 'Component': self.dataset.add_components(context=self.context, component=data) elif type == 'Catalogue': - identifier = parser.obtain_id(title) + identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) - - print(identifier) elif type == 'Observation': - identifier = parser.obtain_id(title) + identifier = self.parser.obtain_id(title) self.observation.add_data(title=title, observation_id=identifier, data=data) - - print(identifier) elif type == 'Dataset': - identifier = parser.obtain_id(title) + identifier = self.parser.obtain_id(title) self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) self.dataset.add_data(title=title, dataset_id=identifier, data=data) @@ -166,26 +161,26 @@ def create_data(self, type, data, title): elif type == 'Dimension': dimension = Dimension() dimension.add_context(context=self.context, context_mapping=self.context_mapping) - dimension_id = parser.obtain_id(title) + dimension_id = self.parser.obtain_id(title) dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) elif type == 'Attribute': attribute = Attribute() attribute.add_context(context=self.context, context_mapping=self.context_mapping) - attribute_id = parser.obtain_id(title) + attribute_id = self.parser.obtain_id(title) attribute.add_data(attribute_id=attribute_id, data=data) self.attributes.append(attribute) elif type == 'ConceptScheme': concept_schema = ConceptSchema() concept_schema.add_context(context=self.context, context_mapping=self.context_mapping) - concept_schema_id = parser.obtain_id(title) + concept_schema_id = self.parser.obtain_id(title) concept_schema.add_data(concept_schema_id=concept_schema_id, data=data) self.conceptSchemas.append(concept_schema) elif type == 'Class': # We need the Concept because each of the Range description is of the type Concept concept_list = Concept() concept_list.add_context(context=self.context, context_mapping=self.context_mapping) - concept_list_id = parser.obtain_id(title) + concept_list_id = self.parser.obtain_id(title) concept_list.add_data(concept_id=concept_list_id, data=data) self.conceptLists.append(concept_list) self.conceptListsIds[title] = concept_list.get_id() @@ -193,7 +188,7 @@ def create_data(self, type, data, title): # TODO: Range is associated to a Concept and identified properly in the ConceptSchema data_range = Concept() data_range.add_context(context=self.context, context_mapping=self.context_mapping) - data_range_id = parser.obtain_id(title) + data_range_id = self.parser.obtain_id(title) data_range.add_data(concept_id=data_range_id, data=data) self.conceptLists.append(data_range) self.conceptListsIds[title] = data_range.get_id() From 7eb4828b0bc7c09dc36f6de78b1c39750932970e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 16 Jun 2023 09:25:10 +0200 Subject: [PATCH 32/74] Implement multi observations support --- examples/observation.ttl | 46 ++++++++++++++++++++++++++++ sdmx2jsonld/transform/entitytype.py | 35 +++++++++++---------- sdmx2jsonld/transform/parser.py | 6 ++-- sdmx2jsonld/transform/transformer.py | 17 +++++----- 4 files changed, 79 insertions(+), 25 deletions(-) diff --git a/examples/observation.ttl b/examples/observation.ttl index f7cc9b8..fc32d12 100644 --- a/examples/observation.ttl +++ b/examples/observation.ttl @@ -23,6 +23,29 @@ qb:structure ; rdfs:label "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr ; sdmx-attribute:title "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr . + a qb:Observation; + ; + "W2" ; + "S1" ; + "S1" ; + "B" ; + "B1G" ; + "_Z" ; + "A" ; + "_Z" ; + "XDC" ; + "V" ; + "N" ; +qb:dataSet ; +sdmx-attribute:confStatus sdmx-code:confStatus-F ; +sdmx-attribute:decimals sdmx-code:decimals-1 ; +sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; +sdmx-attribute:unitMult sdmx-code:unitMult-6 ; +sdmx-dimension:freq sdmx-code:freq-A ; +sdmx-dimension:refArea "BE" ; +sdmx-dimension:timePeriod "2011" ; +sdmx-measure:obsValue "1016.9"^^xsd:float . + a qb:Observation; ; "W2" ; @@ -45,3 +68,26 @@ sdmx-dimension:freq sdmx-code:freq-A ; sdmx-dimension:refArea "BE" ; sdmx-dimension:timePeriod "2012" ; sdmx-measure:obsValue "3016.9"^^xsd:float . + + a qb:Observation; + ; + "W2" ; + "S1" ; + "S1" ; + "B" ; + "B1G" ; + "_Z" ; + "A" ; + "_Z" ; + "XDC" ; + "V" ; + "N" ; +qb:dataSet ; +sdmx-attribute:confStatus sdmx-code:confStatus-F ; +sdmx-attribute:decimals sdmx-code:decimals-1 ; +sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; +sdmx-attribute:unitMult sdmx-code:unitMult-6 ; +sdmx-dimension:freq sdmx-code:freq-A ; +sdmx-dimension:refArea "BE" ; +sdmx-dimension:timePeriod "2013" ; +sdmx-measure:obsValue "9016.9"^^xsd:float . diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 79a1f81..57be2f7 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -60,7 +60,7 @@ def __init__(self): self.context = dict() self.context_mapping = dict() self.catalogue = CatalogueDCATAP() - self.observation = Observation() + self.observations = list() self.pre = Precedence() self.parser = RegParser() @@ -117,7 +117,7 @@ def transform(self, string): data_type, new_string, is_new = self.__find_entity_type__(string=string) if is_new: - self.create_data(type=data_type, data=new_string, title=string[0]) + self.create_data(entity_type=data_type, data=new_string, title=string[0]) else: logger.info(f'Checking previous subjects to find if it was created previously') self.patch_data(datatype=data_type, data=new_string) @@ -142,41 +142,44 @@ def flatten_value(y): if datatype == 'Dataset': self.dataset.patch_data(data=flatten_data, language_map=language_map) - def create_data(self, type, data, title): - if type == 'Component': + def create_data(self, entity_type, data, title): + if entity_type == 'Component': self.dataset.add_components(context=self.context, component=data) - elif type == 'Catalogue': + elif entity_type == 'Catalogue': identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) - elif type == 'Observation': + elif entity_type == 'Observation': + observation = Observation() identifier = self.parser.obtain_id(title) - self.observation.add_data(title=title, observation_id=identifier, data=data) - elif type == 'Dataset': + observation.add_data(title=title, observation_id=identifier, data=data) + self.observations.append(observation) + # self.observation.add_data(title=title, observation_id=identifier, data=data) + elif entity_type == 'Dataset': identifier = self.parser.obtain_id(title) self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) self.dataset.add_data(title=title, dataset_id=identifier, data=data) # Create the CatalogueDCAT-AP and assign the dataset id self.catalogue.add_dataset(dataset_id=self.dataset.data['id']) - elif type == 'Dimension': + elif entity_type == 'Dimension': dimension = Dimension() dimension.add_context(context=self.context, context_mapping=self.context_mapping) dimension_id = self.parser.obtain_id(title) dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) - elif type == 'Attribute': + elif entity_type == 'Attribute': attribute = Attribute() attribute.add_context(context=self.context, context_mapping=self.context_mapping) attribute_id = self.parser.obtain_id(title) attribute.add_data(attribute_id=attribute_id, data=data) self.attributes.append(attribute) - elif type == 'ConceptScheme': + elif entity_type == 'ConceptScheme': concept_schema = ConceptSchema() concept_schema.add_context(context=self.context, context_mapping=self.context_mapping) concept_schema_id = self.parser.obtain_id(title) concept_schema.add_data(concept_schema_id=concept_schema_id, data=data) self.conceptSchemas.append(concept_schema) - elif type == 'Class': + elif entity_type == 'Class': # We need the Concept because each of the Range description is of the type Concept concept_list = Concept() concept_list.add_context(context=self.context, context_mapping=self.context_mapping) @@ -184,7 +187,7 @@ def create_data(self, type, data, title): concept_list.add_data(concept_id=concept_list_id, data=data) self.conceptLists.append(concept_list) self.conceptListsIds[title] = concept_list.get_id() - elif type == 'Range': + elif entity_type == 'Range': # TODO: Range is associated to a Concept and identified properly in the ConceptSchema data_range = Concept() data_range.add_context(context=self.context, context_mapping=self.context_mapping) @@ -212,7 +215,7 @@ def get_catalogue(self): return self.catalogue.get() def get_observation(self): - return self.observation.get() + return self.observations def get_dataset(self): return self.dataset.get() @@ -223,10 +226,10 @@ def get_dimensions(self): def get_attributes(self): return self.attributes - def get_conceptSchemas(self): + def get_concept_schemas(self): return self.conceptSchemas - def get_conceptList(self): + def get_concept_list(self): return self.conceptLists def set_context(self, context, mapping): diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index f60397f..d950333 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -86,12 +86,13 @@ def parsing_file(self, content: TextIOWrapper, out: bool): print() pprint(transform.get_catalogue()) - self.__check_pprint__(transform.get_observation()) + # self.__check_pprint__(transform.get_observation()) self.__check_pprint__(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] [pprint(x.get()) for x in transform.get_concept_schemas()] [pprint(x.get()) for x in transform.get_concept_lists()] + [pprint(x.get()) for x in transform.get_observation()] def parsing_string(self, content: StringIO): transform = TreeToJson() @@ -105,12 +106,13 @@ def parsing_string(self, content: StringIO): # Serializing json payload result = list() result.append(transform.get_catalogue()) - result.append(transform.get_observation()) + # result.append(transform.get_observation()) result.append(transform.get_dataset()) [result.append(x.get()) for x in transform.get_dimensions()] [result.append(x.get()) for x in transform.get_attributes()] [result.append(x.get()) for x in transform.get_concept_schemas()] [result.append(x.get()) for x in transform.get_concept_lists()] + [result.append(x.get()) for x in transform.get_observation()] json_object = dumps(result, indent=4, ensure_ascii=False) diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index 0fed610..ef9fca8 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -116,7 +116,7 @@ def get_catalogue(self): return self.entity_type.get_catalogue() def get_observation(self): - if self.entity_type.observation.data['id'] != '': + if self.entity_type.observations.data['id'] != '': return self.entity_type.get_observation() def get_dataset(self): @@ -130,16 +130,16 @@ def get_attributes(self): return self.entity_type.get_attributes() def get_concept_schemas(self): - return self.entity_type.get_conceptSchemas() + return self.entity_type.get_concept_schemas() def get_concept_lists(self): - return self.entity_type.get_conceptList() + return self.entity_type.get_concept_list() def save(self): self.entity_type.save('catalogue') - if self.entity_type.observation.data['id'] != '': - self.entity_type.save('observation') + #if len(self.entity_type.observations) != 0: + # self.entity_type.save('observation') if self.entity_type.dataset.data['id'] != '': self.entity_type.save('dataset') @@ -150,9 +150,12 @@ def save(self): attributes = self.entity_type.get_attributes() [attribute.save() for attribute in attributes] - concept_schemas = self.entity_type.get_conceptSchemas() + concept_schemas = self.entity_type.get_concept_schemas() [x.save() for x in concept_schemas] - concept_lists = self.entity_type.get_conceptList() + concept_lists = self.entity_type.get_concept_list() [x.save() for x in concept_lists] + if len(self.entity_type.observations) != 0: + observations = self.entity_type.get_observation() + [observation.save() for observation in observations] From c56e4c480f2a6fc3eb0d9fa4a22d43f6a227301e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 16 Jun 2023 12:02:05 +0200 Subject: [PATCH 33/74] Adding sdmx-dimensions properties --- sdmx2jsonld/common/__init__.py | 0 sdmx2jsonld/sdmxattributes/__init__.py | 0 sdmx2jsonld/sdmxattributes/frequency.py | 20 ----- sdmx2jsonld/sdmxdimensions/__init__.py | 0 sdmx2jsonld/sdmxdimensions/frequency.py | 97 ++++++++++++++++++++++++ sdmx2jsonld/sdmxdimensions/refarea.py | 80 +++++++++++++++++++ sdmx2jsonld/sdmxdimensions/timeperiod.py | 97 ++++++++++++++++++++++++ sdmx2jsonld/transform/dataset.py | 14 ++++ sdmx2jsonld/transform/entitytype.py | 6 +- sdmx2jsonld/transform/observation.py | 2 +- sdmx2jsonld/transform/parser.py | 2 - sdmx2jsonld/transform/transformer.py | 3 - 12 files changed, 293 insertions(+), 28 deletions(-) create mode 100644 sdmx2jsonld/common/__init__.py create mode 100644 sdmx2jsonld/sdmxattributes/__init__.py delete mode 100644 sdmx2jsonld/sdmxattributes/frequency.py create mode 100644 sdmx2jsonld/sdmxdimensions/__init__.py create mode 100644 sdmx2jsonld/sdmxdimensions/frequency.py create mode 100644 sdmx2jsonld/sdmxdimensions/refarea.py create mode 100644 sdmx2jsonld/sdmxdimensions/timeperiod.py diff --git a/sdmx2jsonld/common/__init__.py b/sdmx2jsonld/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/sdmxattributes/__init__.py b/sdmx2jsonld/sdmxattributes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/sdmxattributes/frequency.py b/sdmx2jsonld/sdmxattributes/frequency.py deleted file mode 100644 index 4c56206..0000000 --- a/sdmx2jsonld/sdmxattributes/frequency.py +++ /dev/null @@ -1,20 +0,0 @@ -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError - - -class Frequency: - def fix_value(self, value): - # Need to check if the value received is in the list of possible values -> return that value - # then maybe could be in the form freq-, so we have to extract the substring and - # return that substring if it is in the list of values, if not return an error. - # any other value will return an error - value_upper = value.upper() - - m = search('FREQ-(.*)', value_upper) - - if m is not None: - status = m.group(1) - return status - else: - # We received a value that it is not following the template format - raise ClassFreqError(value) diff --git a/sdmx2jsonld/sdmxdimensions/__init__.py b/sdmx2jsonld/sdmxdimensions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py new file mode 100644 index 0000000..2354078 --- /dev/null +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class Frequency(CommonClass): + def __init__(self): + # sdmx-dimension:freq a qb:DimensionProperty, rdf:Property; + # rdfs:range rdfs:Resource; + # qb:concept sdmx-concept:freq; + # rdfs:label "Frequency"@en; + # rdfs:comment """The time interval at which observations occur over a given time period."""@en; + # rdfs:isDefinedBy . + super().__init__(entity='DimensionProperty') + self.data = { + "id": "urn:ngsi-ld:DimensionProperty:freq", + "type": "DimensionProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": "Frequency", + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": "The time interval at which observations occur over a given time period.", + } + }, + "conceptDefinedBy": { + "type": "Property", + "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" + "/vocab/sdmx-concept.ttl" + }, + "isDefinedBy": { + "type": "Property", + "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + }, + "dct:identifier": { + "type": "Property", + "value": "freq" + }, + "rdfs:range": { + "type": "Property", + "value": "xsd:string" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } + + @staticmethod + def fix_value(value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form freq-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + value_upper = value.upper() + + m = search('FREQ-(.*)', value_upper) + + if m is not None: + status = m.group(1) + return status + else: + # We received a value that it is not following the template format + raise ClassFreqError(value) diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py new file mode 100644 index 0000000..fc133df --- /dev/null +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class RefArea(CommonClass): + def __init__(self): + # sdmx-dimension:refArea a qb:DimensionProperty, rdf:Property; + # rdfs:range rdfs:Resource; + # qb:concept sdmx-concept:refArea; + # rdfs:label "Reference Area"@en; + # rdfs:comment "The country or geographic area to which the measured statistical phenomenon relates."@en; + # rdfs:isDefinedBy . + super().__init__(entity='DimensionProperty') + self.data = { + "id": "urn:ngsi-ld:DimensionProperty:refArea", + "type": "DimensionProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": "Reference Area", + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": "The country or geographic area to which the measured statistical phenomenon relates.", + } + }, + "conceptDefinedBy": { + "type": "Property", + "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" + "/vocab/sdmx-concept.ttl" + }, + "isDefinedBy": { + "type": "Property", + "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + }, + "dct:identifier": { + "type": "Property", + "value": "refArea" + }, + "rdfs:range": { + "type": "Property", + "value": "xsd:string" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py new file mode 100644 index 0000000..9757041 --- /dev/null +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class TimePeriod(CommonClass): + def __init__(self): + # sdmx-dimension:timePeriod a qb:DimensionProperty, rdf:Property; + # rdfs:range rdfs:Resource; + # qb:concept sdmx-concept:timePeriod; + # rdfs:label "Time Period"@en; + # rdfs:comment "The period of time or point in time to which the measured observation refers."@en; + # rdfs:isDefinedBy . + super().__init__(entity='DimensionProperty') + self.data = { + "id": "urn:ngsi-ld:DimensionProperty:timePeriod", + "type": "DimensionProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": "Time Period", + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": "The period of time or point in time to which the measured observation refers.", + } + }, + "conceptDefinedBy": { + "type": "Property", + "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" + "/vocab/sdmx-concept.ttl" + }, + "isDefinedBy": { + "type": "Property", + "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + }, + "dct:identifier": { + "type": "Property", + "value": "timePeriod" + }, + "rdfs:range": { + "type": "Property", + "value": "xsd:string" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } + + @staticmethod + def fix_value(value): + # Need to check if the value received is in the list of possible values -> return that value + # then maybe could be in the form freq-, so we have to extract the substring and + # return that substring if it is in the list of values, if not return an error. + # any other value will return an error + value_upper = value.upper() + + m = search('FREQ-(.*)', value_upper) + + if m is not None: + status = m.group(1) + return status + else: + # We received a value that it is not following the template format + raise ClassFreqError(value) diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 02b2467..b1f7bb7 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -24,6 +24,9 @@ from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.common.listmanagement import get_rest_data from sdmx2jsonld.transform.context import Context +from sdmx2jsonld.sdmxdimensions.frequency import Frequency +from sdmx2jsonld.sdmxdimensions.refarea import RefArea +from sdmx2jsonld.sdmxdimensions.timeperiod import TimePeriod logger = getLogger() @@ -99,6 +102,12 @@ def __init__(self): {self.components['qb:dimension']['key']: self.components['qb:dimension']['key']} | \ {self.components['qb:measure']['key']: self.components['qb:measure']['key']} + self.sdmx_dimensions = { + "freq": Frequency(), + "refArea": RefArea(), + "timePeriod": TimePeriod() + } + def add_components(self, context, component): # We need to know which kind of component we have, it should be the verb: # qb:attribute, qb:dimension, or qb:measure @@ -125,6 +134,9 @@ def add_components(self, context, component): logger.warning( f"The component {name} is defined probably outside of the file, " f"creating manually the DimensionsProperty entity") + self.components[type_component]['value'][key]['object'].append(new_id) + self.data = self.data | self.components[type_component]['value'] + return self.sdmx_dimensions[name] else: self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] @@ -142,6 +154,8 @@ def add_components(self, context, component): a.order_context() self.data = a.get_data() + return None + def get(self): return self.data diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 57be2f7..c6b5b69 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -144,7 +144,10 @@ def flatten_value(y): def create_data(self, entity_type, data, title): if entity_type == 'Component': - self.dataset.add_components(context=self.context, component=data) + some_new_dimensions = self.dataset.add_components(context=self.context, component=data) + if some_new_dimensions is not None: + # we have found special sdmx_dimensions that we have to add to dimensions list + self.dimensions.append(some_new_dimensions) elif entity_type == 'Catalogue': identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) @@ -153,7 +156,6 @@ def create_data(self, entity_type, data, title): identifier = self.parser.obtain_id(title) observation.add_data(title=title, observation_id=identifier, data=data) self.observations.append(observation) - # self.observation.add_data(title=title, observation_id=identifier, data=data) elif entity_type == 'Dataset': identifier = self.parser.obtain_id(title) self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index ab7cf3d..a25345a 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -25,7 +25,7 @@ from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from sdmx2jsonld.sdmxattributes.code import Code -from sdmx2jsonld.sdmxattributes.frequency import Frequency +from sdmx2jsonld.sdmxdimensions.frequency import Frequency from sdmx2jsonld.common.regparser import RegParser from re import search, compile diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index d950333..41021c5 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -86,7 +86,6 @@ def parsing_file(self, content: TextIOWrapper, out: bool): print() pprint(transform.get_catalogue()) - # self.__check_pprint__(transform.get_observation()) self.__check_pprint__(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] @@ -106,7 +105,6 @@ def parsing_string(self, content: StringIO): # Serializing json payload result = list() result.append(transform.get_catalogue()) - # result.append(transform.get_observation()) result.append(transform.get_dataset()) [result.append(x.get()) for x in transform.get_dimensions()] [result.append(x.get()) for x in transform.get_attributes()] diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index ef9fca8..47dac53 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -138,9 +138,6 @@ def get_concept_lists(self): def save(self): self.entity_type.save('catalogue') - #if len(self.entity_type.observations) != 0: - # self.entity_type.save('observation') - if self.entity_type.dataset.data['id'] != '': self.entity_type.save('dataset') From 797305e7d1beff38aeb1800b5980933541686bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 16 Jun 2023 19:31:33 +0200 Subject: [PATCH 34/74] Add SDMX-Concepts, SDMX-Dimensions, SDMX-ConceptSchemas, and DCAT-AP:Distribution --- api/custom_logging.py | 14 ++- sdmx2jsonld/sdmxconcepts/__init__.py | 0 sdmx2jsonld/sdmxconcepts/cogconceptschema.py | 54 +++++++++++ sdmx2jsonld/sdmxconcepts/freqconcept.py | 65 +++++++++++++ sdmx2jsonld/sdmxconcepts/refareaconcept.py | 66 +++++++++++++ sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 65 +++++++++++++ sdmx2jsonld/sdmxdimensions/frequency.py | 11 +-- sdmx2jsonld/sdmxdimensions/refarea.py | 11 +-- sdmx2jsonld/sdmxdimensions/timeperiod.py | 11 +-- sdmx2jsonld/transform/catalogue.py | 4 +- sdmx2jsonld/transform/dataset.py | 23 ++++- sdmx2jsonld/transform/distribution.py | 94 +++++++++++++++++++ sdmx2jsonld/transform/entitytype.py | 11 ++- sdmx2jsonld/transform/parser.py | 28 +++++- sdmx2jsonld/transform/transformer.py | 8 ++ 15 files changed, 422 insertions(+), 43 deletions(-) create mode 100644 sdmx2jsonld/sdmxconcepts/__init__.py create mode 100644 sdmx2jsonld/sdmxconcepts/cogconceptschema.py create mode 100644 sdmx2jsonld/sdmxconcepts/freqconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/refareaconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/timeperiodconcept.py create mode 100644 sdmx2jsonld/transform/distribution.py diff --git a/api/custom_logging.py b/api/custom_logging.py index 4465e1f..4f35224 100644 --- a/api/custom_logging.py +++ b/api/custom_logging.py @@ -53,14 +53,12 @@ def emit(self, record): log.opt( depth=depth, exception=record.exc_info - ).log(level,record.getMessage()) + ).log(level, record.getMessage()) class CustomizeLogger: - @classmethod def make_logger(cls, config_path: Path): - config = cls.load_logging_config(config_path) logging_config = config.get('logger') @@ -75,11 +73,11 @@ def make_logger(cls, config_path: Path): @classmethod def customize_logging(cls, - filepath: Path, - level: str, - rotation: str, - retention: str, - format: str): + filepath: Path, + level: str, + rotation: str, + retention: str, + format: str): logger.remove() diff --git a/sdmx2jsonld/sdmxconcepts/__init__.py b/sdmx2jsonld/sdmxconcepts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py new file mode 100644 index 0000000..87f27f7 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class CogConceptSchema(CommonClass): + def __init__(self): + # sdmx-concept:cog a skos:ConceptScheme; + # rdfs:label "Content Oriented Guidelines concept scheme"@en; + # rdfs:isDefinedBy . + super().__init__(entity='ConceptSchema') + + self.data = { + "id": "urn:ngsi-ld:ConceptSchema:cog", + "type": "ConceptScheme", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Content Oriented Guidelines concept scheme" + } + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py new file mode 100644 index 0000000..039720b --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class FreqConcept(CommonClass): + def __init__(self): + # sdmx-concept:freq a sdmx:Concept, skos:Concept; + # rdfs:label "Frequency"@en; + # rdfs:comment """The time interval at which observations occur over a given time period."""@en; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ"; + # skos:inScheme sdmx-concept:cog. + super().__init__(entity='Concept') + + self.data = { + "id": "urn:ngsi-ld:Concept:freq", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Frequency" + } + }, + "skos:notation": { + "type": "Property", + "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py new file mode 100644 index 0000000..05abfb4 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class RefAreaConcept(CommonClass): + def __init__(self): + # sdmx-concept:refArea a sdmx:Concept, skos:Concept ; + # rdfs:label "Reference Area"@en ; + # rdfs:comment """The country or geographic area to which the measured statistical phenomenon relates."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA"; + # skos:inScheme sdmx-concept:cog . + + super().__init__(entity='Concept') + + self.data = { + "id": "urn:ngsi-ld:Concept:refArea", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Reference Area" + } + }, + "skos:notation": { + "type": "Property", + "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py new file mode 100644 index 0000000..d072f07 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.common.commonclass import CommonClass + + +class TimePeriodConcept(CommonClass): + def __init__(self): + # sdmx-concept:timePeriod a sdmx:Concept, skos:Concept; + # rdfs:label "Time Period"@en; + # rdfs:comment """The period of time or point in time to which the measured observation refers."""@en; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; + # skos:inScheme sdmx-concept:cog. + super().__init__(entity='Concept') + + self.data = { + "id": "urn:ngsi-ld:Concept:timePeriod", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Time Period" + } + }, + "skos:notation": { + "type": "Property", + "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py index 2354078..885b085 100644 --- a/sdmx2jsonld/sdmxdimensions/frequency.py +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -52,14 +52,9 @@ def __init__(self): "en": "The time interval at which observations occur over a given time period.", } }, - "conceptDefinedBy": { - "type": "Property", - "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" - "/vocab/sdmx-concept.ttl" - }, - "isDefinedBy": { - "type": "Property", - "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + "concept": { + "type": "Relationship", + "object": "urn:ngsi-ld:Concept:freq" }, "dct:identifier": { "type": "Property", diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index fc133df..27fb21d 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -52,14 +52,9 @@ def __init__(self): "en": "The country or geographic area to which the measured statistical phenomenon relates.", } }, - "conceptDefinedBy": { - "type": "Property", - "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" - "/vocab/sdmx-concept.ttl" - }, - "isDefinedBy": { - "type": "Property", - "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + "concept": { + "type": "Relationship", + "object": "urn:ngsi-ld:Concept:refArea" }, "dct:identifier": { "type": "Property", diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py index 9757041..12a7b8b 100644 --- a/sdmx2jsonld/sdmxdimensions/timeperiod.py +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -52,14 +52,9 @@ def __init__(self): "en": "The period of time or point in time to which the measured observation refers.", } }, - "conceptDefinedBy": { - "type": "Property", - "value": "https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main" - "/vocab/sdmx-concept.ttl" - }, - "isDefinedBy": { - "type": "Property", - "value": "https://sdmx.org/wp-content/uploads/01_sdmx_cog_annex_1_cdc_2009.pdf" + "concept": { + "type": "Relationship", + "object": "urn:ngsi-ld:Concept:timePeriod" }, "dct:identifier": { "type": "Property", diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index e49defb..d07307a 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -24,7 +24,7 @@ from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.common.listmanagement import get_rest_data from sdmx2jsonld.transform.context import Context -import random +from random import getrandbits logger = getLogger() @@ -87,7 +87,7 @@ def add_dataset(self, dataset_id): self.concept_id = dataset_id # generate hash id - random_bits = random.getrandbits(128) + random_bits = getrandbits(128) hash1 = "%032x" % random_bits # Add the id diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index b1f7bb7..b79fa07 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -27,6 +27,10 @@ from sdmx2jsonld.sdmxdimensions.frequency import Frequency from sdmx2jsonld.sdmxdimensions.refarea import RefArea from sdmx2jsonld.sdmxdimensions.timeperiod import TimePeriod +from sdmx2jsonld.sdmxconcepts.freqconcept import FreqConcept +from sdmx2jsonld.sdmxconcepts.cogconceptschema import CogConceptSchema +from sdmx2jsonld.sdmxconcepts.timeperiodconcept import TimePeriodConcept +from sdmx2jsonld.sdmxconcepts.refareaconcept import RefAreaConcept logger = getLogger() @@ -108,6 +112,14 @@ def __init__(self): "timePeriod": TimePeriod() } + self.sdmx_concepts = { + "freq": FreqConcept(), + "refArea": RefAreaConcept(), + "timePeriod": TimePeriodConcept() + } + + self.sdmx_concept_schemas = CogConceptSchema() + def add_components(self, context, component): # We need to know which kind of component we have, it should be the verb: # qb:attribute, qb:dimension, or qb:measure @@ -130,13 +142,18 @@ def add_components(self, context, component): logger.warning( f"The component {new_id} is duplicated and already defined in the {self.data['id']}") elif name in list_special_dimensions: - # We need to create manually the description of these dimensions + # We need to create manually the description of these dimensions, concepts, and conceptschemas logger.warning( f"The component {name} is defined probably outside of the file, " f"creating manually the DimensionsProperty entity") self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] - return self.sdmx_dimensions[name] + + new_dimension = self.sdmx_dimensions[name] + new_concept = self.sdmx_concepts[name] + new_concept_schema = self.sdmx_concept_schemas + + return new_dimension, new_concept, new_concept_schema else: self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] @@ -154,7 +171,7 @@ def add_components(self, context, component): a.order_context() self.data = a.get_data() - return None + return None, None, None def get(self): return self.data diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py new file mode 100644 index 0000000..d9916a0 --- /dev/null +++ b/sdmx2jsonld/transform/distribution.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from logging import getLogger +from sdmx2jsonld.common.commonclass import CommonClass +from pathlib import Path +from json import load +from random import getrandbits + +logger = getLogger() + + +class Distribution(CommonClass): + def __init__(self): + super().__init__(entity="Distribution") + self.data = { + "id": "urn:ngsi-ld:DistributionDCAT-AP:", + "accessUrl": { + "type": "Property", + "value": [ + "/ngsi-ld/v1/entities?type=https://smartdatamodels.org/dataModel.SDMX/Observation" + ] + }, + "availability": { + "type": "Property", + "value": "STABLE" + }, + "description": { + "type": "Property", + "value": "Distribution of statistical data observations." + }, + "format": { + "type": "Property", + "value": "JSON_LD" + }, + "accessService": { + "type": "Property", + "value": [ + "Orion-LD" + ] + }, + "language": { + "type": "Property", + "value": list() + }, + "status": { + "type": "Property", + "value": "Completed" + }, + "Title": { + "type": "Property", + "value": list() + }, + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" + ] + } + + def generate_data(self, catalogue): + # Generate random id for the distribution + random_bits = getrandbits(128) + hash1 = "%032x" % random_bits + self.data['id'] += hash1 + + # Title is extracted from the dcterms:title from the Catalogue + self.data['Title'] = catalogue.data['dcterms:title']['value'] + + # language es obtained from language from the Catalogue + self.data['language'] = catalogue.data['dct:language']['value'] + + # accessURL is generated from the configuration file. + config_path = Path.cwd().joinpath('common/config.json') + with open(config_path) as config_file: + config = load(config_file) + + self.data['accessUrl']['value'][0] = config['broker'] + self.data['accessUrl']['value'][0] diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index c6b5b69..b44d695 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -144,10 +144,15 @@ def flatten_value(y): def create_data(self, entity_type, data, title): if entity_type == 'Component': - some_new_dimensions = self.dataset.add_components(context=self.context, component=data) - if some_new_dimensions is not None: + some_new_dimension, some_new_concept, some_new_concept_schema = \ + self.dataset.add_components(context=self.context, component=data) + + if some_new_dimension is not None: # we have found special sdmx_dimensions that we have to add to dimensions list - self.dimensions.append(some_new_dimensions) + self.dimensions.append(some_new_dimension) + self.conceptLists.append(some_new_concept) + #self.conceptListsIds[title] = concept_list.get_id() + self.conceptSchemas.append(some_new_concept_schema) elif entity_type == 'Catalogue': identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 41021c5..ddd5b06 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -29,6 +29,7 @@ from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken from sdmx2jsonld.common.rdf import turtle_terse from sdmx2jsonld.common.config import GRAMMARFILE +from sdmx2jsonld.transform.distribution import Distribution logger = getLogger(__name__) @@ -85,13 +86,23 @@ def parsing_file(self, content: TextIOWrapper, out: bool): elif content is not None: print() - pprint(transform.get_catalogue()) + catalogue = transform.get_catalogue() + pprint(catalogue) self.__check_pprint__(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] [pprint(x.get()) for x in transform.get_concept_schemas()] [pprint(x.get()) for x in transform.get_concept_lists()] - [pprint(x.get()) for x in transform.get_observation()] + + observations = transform.get_observation() + if len(observations) != 0: + [pprint(x.get()) for x in observations] + + # If we have several observations, we need to generate the DCAT-AP:Distribution class + distribution = Distribution() + distribution.generate_data(catalogue=catalogue) + + pprint(distribution) def parsing_string(self, content: StringIO): transform = TreeToJson() @@ -104,7 +115,8 @@ def parsing_string(self, content: StringIO): # Serializing json payload result = list() - result.append(transform.get_catalogue()) + catalogue = transform.get_catalogue() + result.append(catalogue) result.append(transform.get_dataset()) [result.append(x.get()) for x in transform.get_dimensions()] [result.append(x.get()) for x in transform.get_attributes()] @@ -112,6 +124,16 @@ def parsing_string(self, content: StringIO): [result.append(x.get()) for x in transform.get_concept_lists()] [result.append(x.get()) for x in transform.get_observation()] + observations = transform.get_observation() + if len(observations) != 0: + [result.append(x.get()) for x in observations] + + # If we have several observations, we need to generate the DCAT-AP:Distribution class + distribution = Distribution() + distribution.generate_data(catalogue=catalogue) + + result.append(distribution) + json_object = dumps(result, indent=4, ensure_ascii=False) with open("final.jsonld", "w") as outfile: diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index 47dac53..cc3481a 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -24,6 +24,7 @@ from sdmx2jsonld.transform.context import Context from sdmx2jsonld.transform.entitytype import EntityType from sdmx2jsonld.common.datatypeconversion import DataTypeConversion +from sdmx2jsonld.transform.distribution import Distribution import re @@ -156,3 +157,10 @@ def save(self): if len(self.entity_type.observations) != 0: observations = self.entity_type.get_observation() [observation.save() for observation in observations] + + # If we have several observations, we need to generate the DCAT-AP:Distribution class + distribution = Distribution() + distribution.generate_data(catalogue=self.entity_type.catalogue) + + distribution.save() + From d8ff5326f09db84940977303d116a89c0cf8bcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 19 Jun 2023 10:08:22 +0200 Subject: [PATCH 35/74] Adding the sdmx-attributes obsStatus and confStatus --- ngsild/ngsild_connector.py | 21 ++++++ sdmx2jsonld/common/config.py | 21 ++++++ sdmx2jsonld/common/tzinfos.py | 21 ++++++ sdmx2jsonld/sdmxattributes/code.py | 21 ++++++ .../sdmxattributes/confirmationStatus.py | 74 ++++++++++++++++++- sdmx2jsonld/sdmxattributes/exceptions.py | 21 ++++++ .../sdmxattributes/observationStatus.py | 72 +++++++++++++++++- sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 66 +++++++++++++++++ sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 63 ++++++++++++++++ sdmx2jsonld/transform/dataset.py | 27 +++++-- sdmx2jsonld/transform/entitytype.py | 21 ++++-- 11 files changed, 416 insertions(+), 12 deletions(-) create mode 100644 sdmx2jsonld/sdmxconcepts/confstatusconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/obsstatusconcept.py diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index 06cd213..322f30d 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from pathlib import Path import json from requests import post, exceptions diff --git a/sdmx2jsonld/common/config.py b/sdmx2jsonld/common/config.py index 84b50b5..2409be9 100644 --- a/sdmx2jsonld/common/config.py +++ b/sdmx2jsonld/common/config.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from os.path import join, exists, dirname, abspath # Settings file is inside Basics directory, therefore I have to go back to the parent directory diff --git a/sdmx2jsonld/common/tzinfos.py b/sdmx2jsonld/common/tzinfos.py index e78012b..4e41b22 100644 --- a/sdmx2jsonld/common/tzinfos.py +++ b/sdmx2jsonld/common/tzinfos.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## whois_timezone_info = { "A": 1 * 3600, "ACDT": 10.5 * 3600, diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 51e651e..4e75652 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from re import search from sdmx2jsonld.sdmxattributes.exceptions import ClassCode diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index 5e4c473..b3529e8 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -1,8 +1,30 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from re import search from sdmx2jsonld.sdmxattributes.exceptions import ClassConfStatusError +from sdmx2jsonld.common.commonclass import CommonClass -class ConfStatus: +class ConfStatus(CommonClass): status: list() = [ "F", "N", @@ -18,6 +40,56 @@ class ConfStatus: "P" ] + def __init__(self): + # sdmx-attribute:confStatus a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:confStatus ; + # rdfs:label "Confidentiality - status"@en ; + # rdfs:comment """Information about the confidentiality status of the object to which this + # attribute is attached."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity='AttributeProperty') + self.data = { + "id": "urn:ngsi-ld:AttributeProperty:confStatus", + "type": "AttributeProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": "Confidentiality - status", + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": "Information about the confidentiality status of the object " + "to which this attribute is attached.", + } + }, + "concept": { + "type": "Relationship", + "object": "urn:ngsi-ld:Concept:confStatus" + }, + "dct:identifier": { + "type": "Property", + "value": "confStatus" + }, + "rdfs:range": { + "type": "Property", + "value": "xsd:string" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } + def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value # then maybe could be in the form confStatus-, so we have to extract the substring and diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/sdmxattributes/exceptions.py index 83aa60a..30127bd 100644 --- a/sdmx2jsonld/sdmxattributes/exceptions.py +++ b/sdmx2jsonld/sdmxattributes/exceptions.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## class ClassSDMXAttributeError(Exception): """Base class for other exceptions""" diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index 8344e1f..51c8bbd 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -1,8 +1,30 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from re import search from sdmx2jsonld.sdmxattributes.exceptions import ClassObsStatusError +from sdmx2jsonld.common.commonclass import CommonClass -class ObsStatus: +class ObsStatus(CommonClass): status: list() = [ "A", "B", @@ -26,6 +48,54 @@ class ObsStatus: "V" ] + def __init__(self): + # sdmx-attribute:obsStatus a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:obsStatus ; + # rdfs:label "Observation Status"@en ; + # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity='AttributeProperty') + self.data = { + "id": "urn:ngsi-ld:AttributeProperty:obsStatus", + "type": "AttributeProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": "Observation Status", + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": "Information on the quality of a value or an unusual or missing value.", + } + }, + "concept": { + "type": "Relationship", + "object": "urn:ngsi-ld:Concept:obsStatus" + }, + "dct:identifier": { + "type": "Property", + "value": "obsStatus" + }, + "rdfs:range": { + "type": "Property", + "value": "xsd:string" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } + def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value # then maybe could be in the form obsStatus-, so we have to extract the substring and diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py new file mode 100644 index 0000000..aa525cb --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.common.commonclass import CommonClass + + +class ConfStatusConcept(CommonClass): + def __init__(self): + # sdmx-concept:confStatus a sdmx:Concept, skos:Concept ; + # rdfs:label "Confidentiality - status"@en ; + # rdfs:comment """Information about the confidentiality status of the object to which this + # attribute is attached."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS"; + # skos:broader sdmx-concept:conf; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity='Concept') + + self.data = { + "id": "urn:ngsi-ld:Concept:confStatus", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Confidentiality - status" + } + }, + "skos:notation": { + "type": "Property", + "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py new file mode 100644 index 0000000..eb27080 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.common.commonclass import CommonClass + + +class ObsStatusConcept(CommonClass): + def __init__(self): + # sdmx-concept:obsStatus a sdmx:Concept, skos:Concept ; + # rdfs:label "Observation Status"@en ; + # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity='Concept') + + self.data = { + "id": "urn:ngsi-ld:Concept:obsStatus", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": "Observation Status" + } + }, + "skos:notation": { + "type": "Property", + "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS" + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index b79fa07..00e6485 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -27,10 +27,14 @@ from sdmx2jsonld.sdmxdimensions.frequency import Frequency from sdmx2jsonld.sdmxdimensions.refarea import RefArea from sdmx2jsonld.sdmxdimensions.timeperiod import TimePeriod +from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus +from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus from sdmx2jsonld.sdmxconcepts.freqconcept import FreqConcept from sdmx2jsonld.sdmxconcepts.cogconceptschema import CogConceptSchema from sdmx2jsonld.sdmxconcepts.timeperiodconcept import TimePeriodConcept from sdmx2jsonld.sdmxconcepts.refareaconcept import RefAreaConcept +from sdmx2jsonld.sdmxconcepts.obsstatusconcept import ObsStatusConcept +from sdmx2jsonld.sdmxconcepts.confstatusconcept import ConfStatusConcept logger = getLogger() @@ -112,10 +116,22 @@ def __init__(self): "timePeriod": TimePeriod() } + self.sdmx_attributes = { + "obsStatus": ObsStatus(), + "confStatus": ConfStatus() + } + + self.sdmx_components = { + "DimensionProperty": self.sdmx_dimensions, + "AttributeProperty": self.sdmx_attributes + } + self.sdmx_concepts = { "freq": FreqConcept(), "refArea": RefAreaConcept(), - "timePeriod": TimePeriodConcept() + "timePeriod": TimePeriodConcept(), + "obsStatus": ObsStatusConcept(), + "confStatus": ConfStatusConcept() } self.sdmx_concept_schemas = CogConceptSchema() @@ -127,7 +143,7 @@ def add_components(self, context, component): # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix - list_special_dimensions = ['freq', 'refArea', 'timePeriod'] + list_special_components = ['freq', 'refArea', 'timePeriod', 'obsStatus', 'confStatus'] type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 @@ -141,15 +157,16 @@ def add_components(self, context, component): if new_id in self.components[type_component]['value'][key]['object']: logger.warning( f"The component {new_id} is duplicated and already defined in the {self.data['id']}") - elif name in list_special_dimensions: + elif name in list_special_components: # We need to create manually the description of these dimensions, concepts, and conceptschemas logger.warning( f"The component {name} is defined probably outside of the file, " - f"creating manually the DimensionsProperty entity") + f"creating manually the {entity} entity") self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] - new_dimension = self.sdmx_dimensions[name] + new_component = self.sdmx_components[entity] + new_dimension = new_component[name] new_concept = self.sdmx_concepts[name] new_concept_schema = self.sdmx_concept_schemas diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index b44d695..d27fa60 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -144,15 +144,26 @@ def flatten_value(y): def create_data(self, entity_type, data, title): if entity_type == 'Component': - some_new_dimension, some_new_concept, some_new_concept_schema = \ + some_new_component, some_new_concept, some_new_concept_schema = \ self.dataset.add_components(context=self.context, component=data) - if some_new_dimension is not None: - # we have found special sdmx_dimensions that we have to add to dimensions list - self.dimensions.append(some_new_dimension) + if some_new_component is not None: + if some_new_component.data['type'] == 'DimensionProperty': + # we have found special sdmx_dimensions that we have to add to dimensions list + self.dimensions.append(some_new_component) + elif some_new_component.data['type'] == 'AttributeProperty': + # we have found special sdmx_attribute that we have to add to attributes list + self.attributes.append(some_new_component) + else: + # You should not be here, reporting error... + logger.error(f'Unexpected entity type, id: {some_new_component.data["idf"]} ' + f'type: {some_new_component.data["type"]}') + self.conceptLists.append(some_new_concept) #self.conceptListsIds[title] = concept_list.get_id() - self.conceptSchemas.append(some_new_concept_schema) + # we need to check that the conceptSchema is not already defined in the structure + if some_new_concept_schema not in self.conceptSchemas: + self.conceptSchemas.append(some_new_concept_schema) elif entity_type == 'Catalogue': identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) From 780b6748258a265af2e8bd262bbb40e2d68bc2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 19 Jun 2023 10:11:51 +0200 Subject: [PATCH 36/74] Refactor exception classes to the proper python package --- sdmx2jsonld/{sdmxattributes => exceptions}/exceptions.py | 0 sdmx2jsonld/sdmxattributes/code.py | 2 +- sdmx2jsonld/sdmxattributes/confirmationStatus.py | 2 +- sdmx2jsonld/sdmxattributes/observationStatus.py | 2 +- sdmx2jsonld/sdmxconcepts/cogconceptschema.py | 2 -- sdmx2jsonld/sdmxconcepts/freqconcept.py | 2 -- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 2 -- sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 2 -- sdmx2jsonld/sdmxdimensions/frequency.py | 2 +- sdmx2jsonld/sdmxdimensions/refarea.py | 2 -- sdmx2jsonld/sdmxdimensions/timeperiod.py | 2 +- tests/sdmxattributes/test_code.py | 2 +- tests/sdmxattributes/test_confirmationStatus.py | 2 +- tests/sdmxattributes/test_observationStatus.py | 2 +- 14 files changed, 8 insertions(+), 18 deletions(-) rename sdmx2jsonld/{sdmxattributes => exceptions}/exceptions.py (100%) diff --git a/sdmx2jsonld/sdmxattributes/exceptions.py b/sdmx2jsonld/exceptions/exceptions.py similarity index 100% rename from sdmx2jsonld/sdmxattributes/exceptions.py rename to sdmx2jsonld/exceptions/exceptions.py diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 4e75652..dbe9374 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -20,7 +20,7 @@ # under the License. ## from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassCode +from sdmx2jsonld.exceptions.exceptions import ClassCode class Code: diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index b3529e8..b532dc7 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -20,7 +20,7 @@ # under the License. ## from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassConfStatusError +from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index 51c8bbd..2ab5d53 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -20,7 +20,7 @@ # under the License. ## from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassObsStatusError +from sdmx2jsonld.exceptions.exceptions import ClassObsStatusError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py index 87f27f7..e2cc765 100644 --- a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py +++ b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index 039720b..9e41b09 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index 05abfb4..f3ce987 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index d072f07..14c2238 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py index 885b085..5f46a54 100644 --- a/sdmx2jsonld/sdmxdimensions/frequency.py +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -20,7 +20,7 @@ # under the License. ## from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.exceptions.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index 27fb21d..eca97f7 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py index 12a7b8b..303eb29 100644 --- a/sdmx2jsonld/sdmxdimensions/timeperiod.py +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -20,7 +20,7 @@ # under the License. ## from re import search -from sdmx2jsonld.sdmxattributes.exceptions import ClassFreqError +from sdmx2jsonld.exceptions.exceptions import ClassFreqError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py index 08f0620..4ff4b54 100644 --- a/tests/sdmxattributes/test_code.py +++ b/tests/sdmxattributes/test_code.py @@ -1,6 +1,6 @@ from unittest import TestCase from sdmx2jsonld.sdmxattributes.code import Code -from sdmx2jsonld.sdmxattributes.exceptions import ClassCode +from sdmx2jsonld.exceptions.exceptions import ClassCode class TestConfStatus(TestCase): diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py index ae0d2d5..d8fcb87 100644 --- a/tests/sdmxattributes/test_confirmationStatus.py +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -1,6 +1,6 @@ from unittest import TestCase from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus -from sdmx2jsonld.sdmxattributes.exceptions import ClassConfStatusError +from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError class TestConfStatus(TestCase): diff --git a/tests/sdmxattributes/test_observationStatus.py b/tests/sdmxattributes/test_observationStatus.py index ecbdf0a..59495e6 100644 --- a/tests/sdmxattributes/test_observationStatus.py +++ b/tests/sdmxattributes/test_observationStatus.py @@ -1,6 +1,6 @@ from unittest import TestCase from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus -from sdmx2jsonld.sdmxattributes.exceptions import ClassObsStatusError +from sdmx2jsonld.exceptions.exceptions import ClassObsStatusError class TestObsStatus(TestCase): From 55a46d5385169bceeaa1a65ab0ba8ea1d603fc23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 19 Jun 2023 15:45:37 +0200 Subject: [PATCH 37/74] Refactoring smdx-attributes and sdmx-dimensions --- .../sdmxattributes/confirmationStatus.py | 94 ++++++++++--------- sdmx2jsonld/sdmxattributes/decimals.py | 78 +++++++++++++++ .../sdmxattributes/observationStatus.py | 92 +++++++++--------- sdmx2jsonld/sdmxattributes/sdmxattribute.py | 69 ++++++++++++++ sdmx2jsonld/sdmxattributes/timeFormat.py | 78 +++++++++++++++ sdmx2jsonld/sdmxattributes/timePerCollect.py | 84 +++++++++++++++++ sdmx2jsonld/sdmxattributes/title.py | 37 ++++++++ sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 67 ++++++------- sdmx2jsonld/sdmxconcepts/decimals.py | 65 +++++++++++++ sdmx2jsonld/sdmxconcepts/freqconcept.py | 67 ++++++------- sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 67 ++++++------- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 67 ++++++------- sdmx2jsonld/sdmxconcepts/sdmxconcept.py | 56 +++++++++++ .../sdmxconcepts/timePerCollectConcept.py | 70 ++++++++++++++ sdmx2jsonld/sdmxconcepts/timeformatconcept.py | 65 +++++++++++++ sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 9 +- sdmx2jsonld/sdmxconcepts/titleConcept.py | 35 +++++++ sdmx2jsonld/transform/dataset.py | 27 +++++- 18 files changed, 906 insertions(+), 221 deletions(-) create mode 100644 sdmx2jsonld/sdmxattributes/decimals.py create mode 100644 sdmx2jsonld/sdmxattributes/sdmxattribute.py create mode 100644 sdmx2jsonld/sdmxattributes/timeFormat.py create mode 100644 sdmx2jsonld/sdmxattributes/timePerCollect.py create mode 100644 sdmx2jsonld/sdmxattributes/title.py create mode 100644 sdmx2jsonld/sdmxconcepts/decimals.py create mode 100644 sdmx2jsonld/sdmxconcepts/sdmxconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/timeformatconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/titleConcept.py diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index b532dc7..f9cec10 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -21,10 +21,10 @@ ## from re import search from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute -class ConfStatus(CommonClass): +class ConfStatus(SDMXAttribute): status: list() = [ "F", "N", @@ -47,48 +47,54 @@ def __init__(self): # rdfs:comment """Information about the confidentiality status of the object to which this # attribute is attached."""@en ; # rdfs:isDefinedBy . - super().__init__(entity='AttributeProperty') - self.data = { - "id": "urn:ngsi-ld:AttributeProperty:confStatus", - "type": "AttributeProperty", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "rdfs:label": { - "type": "Property", - "value": { - "en": "Confidentiality - status", - } - }, - "dct:description": { - "type": "Property", - "value": { - "en": "Information about the confidentiality status of the object " - "to which this attribute is attached.", - } - }, - "concept": { - "type": "Relationship", - "object": "urn:ngsi-ld:Concept:confStatus" - }, - "dct:identifier": { - "type": "Property", - "value": "confStatus" - }, - "rdfs:range": { - "type": "Property", - "value": "xsd:string" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } - } + # super().__init__(entity='AttributeProperty') + super().__init__(entity_id='confStatus', + label='Confidentiality - status', + description='Information about the confidentiality status of the object to which this attribute is attached.', + concept_id='confStatus', + identifier='confStatus', + entity_range='xsd:string') + # self.data = { + # "id": "urn:ngsi-ld:AttributeProperty:confStatus", + # "type": "AttributeProperty", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "rdfs:label": { + # "type": "Property", + # "value": { + # "en": "Confidentiality - status", + # } + # }, + # "dct:description": { + # "type": "Property", + # "value": { + # "en": "Information about the confidentiality status of the object " + # "to which this attribute is attached.", + # } + # }, + # "concept": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:Concept:confStatus" + # }, + # "dct:identifier": { + # "type": "Property", + # "value": "confStatus" + # }, + # "rdfs:range": { + # "type": "Property", + # "value": "xsd:string" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "qb": "http://purl.org/linked-data/cube#", + # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + # } + # } def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value diff --git a/sdmx2jsonld/sdmxattributes/decimals.py b/sdmx2jsonld/sdmxattributes/decimals.py new file mode 100644 index 0000000..63fec71 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/decimals.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class Decimals(SDMXAttribute): + def __init__(self): + # sdmx-attribute:decimals a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:decimals ; + # rdfs:label "Decimals"@en ; + # rdfs:comment """The number of digits of an observation to the right of a decimal point."""@en ; + # rdfs:isDefinedBy . + # super().__init__(entity='AttributeProperty') + super().__init__(entity_id='decimals', + label='Decimals', + description='The number of digits of an observation to the right of a decimal point.', + concept_id='decimals', + identifier='decimals', + entity_range='xsd:integer') + # self.data = { + # "id": "urn:ngsi-ld:AttributeProperty:decimals", + # "type": "AttributeProperty", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "rdfs:label": { + # "type": "Property", + # "value": { + # "en": "Decimals", + # } + # }, + # "dct:description": { + # "type": "Property", + # "value": { + # "en": "The number of digits of an observation to the right of a decimal point.", + # } + # }, + # "concept": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:Concept:decimals" + # }, + # "dct:identifier": { + # "type": "Property", + # "value": "decimals" + # }, + # "rdfs:range": { + # "type": "Property", + # "value": "xsd:integer" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "qb": "http://purl.org/linked-data/cube#", + # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + # } + # } diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index 2ab5d53..fd41efd 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -21,10 +21,10 @@ ## from re import search from sdmx2jsonld.exceptions.exceptions import ClassObsStatusError -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute -class ObsStatus(CommonClass): +class ObsStatus(SDMXAttribute): status: list() = [ "A", "B", @@ -54,47 +54,53 @@ def __init__(self): # rdfs:label "Observation Status"@en ; # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; # rdfs:isDefinedBy . - super().__init__(entity='AttributeProperty') - self.data = { - "id": "urn:ngsi-ld:AttributeProperty:obsStatus", - "type": "AttributeProperty", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "rdfs:label": { - "type": "Property", - "value": { - "en": "Observation Status", - } - }, - "dct:description": { - "type": "Property", - "value": { - "en": "Information on the quality of a value or an unusual or missing value.", - } - }, - "concept": { - "type": "Relationship", - "object": "urn:ngsi-ld:Concept:obsStatus" - }, - "dct:identifier": { - "type": "Property", - "value": "obsStatus" - }, - "rdfs:range": { - "type": "Property", - "value": "xsd:string" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } - } + # super().__init__(entity='AttributeProperty') + super().__init__(entity_id='obsStatus', + label='Observation Status', + description='Information on the quality of a value or an unusual or missing value.', + concept_id='obsStatus', + identifier='obsStatus', + entity_range='xsd:string') + # self.data = { + # "id": "urn:ngsi-ld:AttributeProperty:obsStatus", + # "type": "AttributeProperty", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "rdfs:label": { + # "type": "Property", + # "value": { + # "en": "Observation Status", + # } + # }, + # "dct:description": { + # "type": "Property", + # "value": { + # "en": "Information on the quality of a value or an unusual or missing value.", + # } + # }, + # "concept": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:Concept:obsStatus" + # }, + # "dct:identifier": { + # "type": "Property", + # "value": "obsStatus" + # }, + # "rdfs:range": { + # "type": "Property", + # "value": "xsd:string" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "qb": "http://purl.org/linked-data/cube#", + # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + # } + # } def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value diff --git a/sdmx2jsonld/sdmxattributes/sdmxattribute.py b/sdmx2jsonld/sdmxattributes/sdmxattribute.py new file mode 100644 index 0000000..c2524f8 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/sdmxattribute.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError +from sdmx2jsonld.common.commonclass import CommonClass + + +class SDMXAttribute(CommonClass): + def __init__(self, entity_id, label, description, concept_id, identifier, entity_range): + super().__init__(entity='AttributeProperty') + self.data = { + "id": f"urn:ngsi-ld:AttributeProperty:{entity_id}", + "type": "AttributeProperty", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "rdfs:label": { + "type": "Property", + "value": { + "en": label, + } + }, + "dct:description": { + "type": "Property", + "value": { + "en": description, + } + }, + "concept": { + "type": "Relationship", + "object": f"urn:ngsi-ld:Concept:{concept_id}" + }, + "dct:identifier": { + "type": "Property", + "value": identifier + }, + "rdfs:range": { + "type": "Property", + "value": entity_range + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "qb": "http://purl.org/linked-data/cube#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + } diff --git a/sdmx2jsonld/sdmxattributes/timeFormat.py b/sdmx2jsonld/sdmxattributes/timeFormat.py new file mode 100644 index 0000000..19b3e39 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/timeFormat.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class TimeFormat(SDMXAttribute): + def __init__(self): + # sdmx-attribute:timeFormat a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:timeFormat ; + # rdfs:label "Time Format"@en ; + # rdfs:comment """Technical format in which time is represented for the measured phenomenon."""@en ; + # rdfs:isDefinedBy . + # super().__init__(entity='AttributeProperty') + super().__init__(entity_id='timeFormat', + label='Time Format', + description='Technical format in which time is represented for the measured phenomenon.', + concept_id='timeFormat', + identifier='timeFormat', + entity_range='xsd:string') + # self.data = { + # "id": "urn:ngsi-ld:AttributeProperty:timeFormat", + # "type": "AttributeProperty", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "rdfs:label": { + # "type": "Property", + # "value": { + # "en": "Time Format", + # } + # }, + # "dct:description": { + # "type": "Property", + # "value": { + # "en": "Technical format in which time is represented for the measured phenomenon.", + # } + # }, + # "concept": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:Concept:timeFormat" + # }, + # "dct:identifier": { + # "type": "Property", + # "value": "timeFormat" + # }, + # "rdfs:range": { + # "type": "Property", + # "value": "xsd:string" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "qb": "http://purl.org/linked-data/cube#", + # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + # } + # } diff --git a/sdmx2jsonld/sdmxattributes/timePerCollect.py b/sdmx2jsonld/sdmxattributes/timePerCollect.py new file mode 100644 index 0000000..a56eada --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/timePerCollect.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class TimePerCollect(SDMXAttribute): + def __init__(self): + # sdmx-attribute:timePerCollect a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:timePerCollect ; + # rdfs:label "Time Period - collection"@en ; + # rdfs:comment """Dates or periods during which the observations have been collected + # (such as middle, average or end of period) to compile the indicator + # for the target reference period."""@en ; + # rdfs:isDefinedBy . + # super().__init__(entity='AttributeProperty') + super().__init__(entity_id='timePerCollect', + label='Time Period - collection', + description='Dates or periods during which the observations have been collected ' + '(such as middle, average or end of period) to compile the indicator ' + 'for the target reference period.', + concept_id='timePerCollect', + identifier='timePerCollect', + entity_range='xsd:string') + # self.data = { + # "id": "urn:ngsi-ld:AttributeProperty:timePerCollect", + # "type": "AttributeProperty", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "rdfs:label": { + # "type": "Property", + # "value": { + # "en": "Time Period - collection", + # } + # }, + # "dct:description": { + # "type": "Property", + # "value": { + # "en": "Dates or periods during which the observations have been collected " + # "(such as middle, average or end of period) to compile the indicator " + # "for the target reference period.", + # } + # }, + # "concept": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:Concept:timePerCollect" + # }, + # "dct:identifier": { + # "type": "Property", + # "value": "timePerCollect" + # }, + # "rdfs:range": { + # "type": "Property", + # "value": "xsd:string" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "qb": "http://purl.org/linked-data/cube#", + # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + # } + # } diff --git a/sdmx2jsonld/sdmxattributes/title.py b/sdmx2jsonld/sdmxattributes/title.py new file mode 100644 index 0000000..bcbb5a4 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/title.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class Title(SDMXAttribute): + def __init__(self): + # sdmx-attribute:title a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:title ; + # rdfs:label "Title"@en ; + # rdfs:comment """Textual label used as identification of a statistical object."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='title', + label='Title', + description='Textual label used as identification of a statistical object.', + concept_id='title', + identifier='title', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py index aa525cb..906c6bd 100644 --- a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept -class ConfStatusConcept(CommonClass): +class ConfStatusConcept(SDMXConcept): def __init__(self): # sdmx-concept:confStatus a sdmx:Concept, skos:Concept ; # rdfs:label "Confidentiality - status"@en ; @@ -33,34 +33,37 @@ def __init__(self): # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS"; # skos:broader sdmx-concept:conf; # skos:inScheme sdmx-concept:cog . - super().__init__(entity='Concept') + # super().__init__(entity='Concept') + super().__init__(entity_id='confStatus', + label='Confidentiality - status', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS') - self.data = { - "id": "urn:ngsi-ld:Concept:confStatus", - "type": "Concept", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "skos:inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "skos:prefLabel": { - "type": "Property", - "value": { - "en": "Confidentiality - status" - } - }, - "skos:notation": { - "type": "Property", - "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } - } + # self.data = { + # "id": "urn:ngsi-ld:Concept:confStatus", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Confidentiality - status" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/decimals.py b/sdmx2jsonld/sdmxconcepts/decimals.py new file mode 100644 index 0000000..eac4d8e --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/decimals.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class DecimalsConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:decimals a sdmx:Concept, skos:Concept ; + # rdfs:label "Decimals"@en ; + # rdfs:comment """The number of digits of an observation to the right of a decimal point."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS"; + # skos:inScheme sdmx-concept:cog . + # super().__init__(entity='Concept') + super().__init__(entity_id='decimals', + label='Decimals', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS') + # self.data = { + # "id": "urn:ngsi-ld:Concept:decimals", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Decimals" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index 9e41b09..56e530c 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept -class FreqConcept(CommonClass): +class FreqConcept(SDMXConcept): def __init__(self): # sdmx-concept:freq a sdmx:Concept, skos:Concept; # rdfs:label "Frequency"@en; @@ -30,34 +30,37 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ"; # skos:inScheme sdmx-concept:cog. - super().__init__(entity='Concept') + #super().__init__(entity='Concept') + super().__init__(entity_id='freq', + label='Frequency', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ') - self.data = { - "id": "urn:ngsi-ld:Concept:freq", - "type": "Concept", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "skos:inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "skos:prefLabel": { - "type": "Property", - "value": { - "en": "Frequency" - } - }, - "skos:notation": { - "type": "Property", - "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } - } + # self.data = { + # "id": "urn:ngsi-ld:Concept:freq", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Frequency" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py index eb27080..b872cad 100644 --- a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept -class ObsStatusConcept(CommonClass): +class ObsStatusConcept(SDMXConcept): def __init__(self): # sdmx-concept:obsStatus a sdmx:Concept, skos:Concept ; # rdfs:label "Observation Status"@en ; @@ -30,34 +30,37 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity='Concept') + # super().__init__(entity='Concept') + super().__init__(entity_id='obsStatus', + label='Observation Status', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS') - self.data = { - "id": "urn:ngsi-ld:Concept:obsStatus", - "type": "Concept", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "skos:inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "skos:prefLabel": { - "type": "Property", - "value": { - "en": "Observation Status" - } - }, - "skos:notation": { - "type": "Property", - "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } - } + # self.data = { + # "id": "urn:ngsi-ld:Concept:obsStatus", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Observation Status" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index f3ce987..6706aa8 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept -class RefAreaConcept(CommonClass): +class RefAreaConcept(SDMXConcept): def __init__(self): # sdmx-concept:refArea a sdmx:Concept, skos:Concept ; # rdfs:label "Reference Area"@en ; @@ -31,34 +31,37 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity='Concept') + # super().__init__(entity='Concept') + super().__init__(entity_id='refArea', + label='Reference Area', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA') - self.data = { - "id": "urn:ngsi-ld:Concept:refArea", - "type": "Concept", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "skos:inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "skos:prefLabel": { - "type": "Property", - "value": { - "en": "Reference Area" - } - }, - "skos:notation": { - "type": "Property", - "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } - } + # self.data = { + # "id": "urn:ngsi-ld:Concept:refArea", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Reference Area" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py new file mode 100644 index 0000000..25c1fae --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.common.commonclass import CommonClass + + +class SDMXConcept(CommonClass): + def __init__(self, entity_id, label, notation): + super().__init__(entity='Concept') + self.data = { + "id": f"urn:ngsi-ld:Concept:{entity_id}", + "type": "Concept", + "dct:language": { + "type": "Property", + "value": ["en"] + }, + "skos:inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "skos:prefLabel": { + "type": "Property", + "value": { + "en": label + } + }, + "skos:notation": { + "type": "Property", + "value": notation + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "dct": "http://purl.org/dc/terms/", + "skos": "http://www.w3.org/2004/02/skos/core#" + } + } diff --git a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py new file mode 100644 index 0000000..09f9082 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class TimePerCollectConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:timePerCollect a sdmx:Concept, skos:Concept ; + # rdfs:label "Time Period - collection"@en ; + # rdfs:comment """Dates or periods during which the observations have been collected + # (such as middle, average or end of period) to compile the indicator + # for the target reference period."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT"; + # skos:broader sdmx-concept:timePeriod; + # skos:inScheme sdmx-concept:cog . + # super().__init__(entity='Concept') + super().__init__(entity_id='timePerCollect', + label='Time Period - collection', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT') + # self.data = { + # "id": "urn:ngsi-ld:Concept:timePerCollect", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Time Period - collection" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py new file mode 100644 index 0000000..f91def1 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class TimeFormatConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:timeFormat a sdmx:Concept, skos:Concept ; + # rdfs:label "Time Format"@en ; + # rdfs:comment """Technical format in which time is represented for the measured phenomenon."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT"; + # skos:inScheme sdmx-concept:cog . + # super().__init__(entity='Concept') + super().__init__(entity_id='timeFormat', + label='Time Format', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT') + # self.data = { + # "id": "urn:ngsi-ld:Concept:timeFormat", + # "type": "Concept", + # "dct:language": { + # "type": "Property", + # "value": ["en"] + # }, + # "skos:inScheme": { + # "type": "Relationship", + # "object": "urn:ngsi-ld:ConceptSchema:cog" + # }, + # "skos:prefLabel": { + # "type": "Property", + # "value": { + # "en": "Time Format" + # } + # }, + # "skos:notation": { + # "type": "Property", + # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT" + # }, + # "@context": { + # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + # "dcat": "http://www.w3.org/ns/dcat#", + # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + # "dct": "http://purl.org/dc/terms/", + # "skos": "http://www.w3.org/2004/02/skos/core#" + # } + # } diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index 14c2238..52e1839 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept -class TimePeriodConcept(CommonClass): +class TimePeriodConcept(SDMXConcept): def __init__(self): # sdmx-concept:timePeriod a sdmx:Concept, skos:Concept; # rdfs:label "Time Period"@en; @@ -30,7 +30,10 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; # skos:inScheme sdmx-concept:cog. - super().__init__(entity='Concept') + #super().__init__(entity='Concept') + super().__init__(entity_id='timePeriod', + label='Time Period', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD') self.data = { "id": "urn:ngsi-ld:Concept:timePeriod", diff --git a/sdmx2jsonld/sdmxconcepts/titleConcept.py b/sdmx2jsonld/sdmxconcepts/titleConcept.py new file mode 100644 index 0000000..cde7540 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/titleConcept.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class TitleConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:title a sdmx:Concept, skos:Concept ; + # rdfs:label "Title"@en ; + # rdfs:comment """Textual label used as identification of a statistical object."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TITLE"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='title', + label='Title', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TITLE') diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 00e6485..3edff8a 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -24,17 +24,28 @@ from sdmx2jsonld.common.commonclass import CommonClass from sdmx2jsonld.common.listmanagement import get_rest_data from sdmx2jsonld.transform.context import Context + from sdmx2jsonld.sdmxdimensions.frequency import Frequency from sdmx2jsonld.sdmxdimensions.refarea import RefArea from sdmx2jsonld.sdmxdimensions.timeperiod import TimePeriod + from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus +from sdmx2jsonld.sdmxattributes.timeFormat import TimeFormat +from sdmx2jsonld.sdmxattributes.timePerCollect import TimePerCollect +from sdmx2jsonld.sdmxattributes.decimals import Decimals +from sdmx2jsonld.sdmxattributes.title import Title + from sdmx2jsonld.sdmxconcepts.freqconcept import FreqConcept from sdmx2jsonld.sdmxconcepts.cogconceptschema import CogConceptSchema from sdmx2jsonld.sdmxconcepts.timeperiodconcept import TimePeriodConcept from sdmx2jsonld.sdmxconcepts.refareaconcept import RefAreaConcept from sdmx2jsonld.sdmxconcepts.obsstatusconcept import ObsStatusConcept from sdmx2jsonld.sdmxconcepts.confstatusconcept import ConfStatusConcept +from sdmx2jsonld.sdmxconcepts.timeformatconcept import TimeFormatConcept +from sdmx2jsonld.sdmxconcepts.timePerCollectConcept import TimePerCollectConcept +from sdmx2jsonld.sdmxconcepts.decimals import DecimalsConcept +from sdmx2jsonld.sdmxconcepts.titleConcept import TitleConcept logger = getLogger() @@ -118,7 +129,11 @@ def __init__(self): self.sdmx_attributes = { "obsStatus": ObsStatus(), - "confStatus": ConfStatus() + "confStatus": ConfStatus(), + "timeFormat": TimeFormat(), + "timePerCollect": TimePerCollect(), + "decimals": Decimals(), + "title": Title() } self.sdmx_components = { @@ -131,7 +146,11 @@ def __init__(self): "refArea": RefAreaConcept(), "timePeriod": TimePeriodConcept(), "obsStatus": ObsStatusConcept(), - "confStatus": ConfStatusConcept() + "confStatus": ConfStatusConcept(), + "timeFormat": TimeFormatConcept(), + "timePerCollect": TimePerCollectConcept(), + "decimals": DecimalsConcept(), + "title": TitleConcept() } self.sdmx_concept_schemas = CogConceptSchema() @@ -143,7 +162,9 @@ def add_components(self, context, component): # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix - list_special_components = ['freq', 'refArea', 'timePeriod', 'obsStatus', 'confStatus'] + list_special_components = ['freq', 'refArea', 'timePeriod', + 'obsStatus', 'confStatus', 'timeFormat', + 'timePerCollect', 'decimals', 'title'] type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 From db01933a3437c9772474d5cd42d133155b8fd52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 19 Jun 2023 15:52:13 +0200 Subject: [PATCH 38/74] Refactoring finished for sdmx-attributes and sdmx-dimensions --- .../sdmxattributes/confirmationStatus.py | 42 ------------------ sdmx2jsonld/sdmxattributes/decimals.py | 41 ------------------ .../sdmxattributes/observationStatus.py | 41 ------------------ sdmx2jsonld/sdmxattributes/timeFormat.py | 41 ------------------ sdmx2jsonld/sdmxattributes/timePerCollect.py | 43 ------------------- sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 31 ------------- sdmx2jsonld/sdmxconcepts/decimals.py | 30 ------------- sdmx2jsonld/sdmxconcepts/freqconcept.py | 31 ------------- sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 31 ------------- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 32 -------------- .../sdmxconcepts/timePerCollectConcept.py | 31 ------------- sdmx2jsonld/sdmxconcepts/timeformatconcept.py | 30 ------------- sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 31 ------------- 13 files changed, 455 deletions(-) diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index f9cec10..6defb97 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -47,54 +47,12 @@ def __init__(self): # rdfs:comment """Information about the confidentiality status of the object to which this # attribute is attached."""@en ; # rdfs:isDefinedBy . - # super().__init__(entity='AttributeProperty') super().__init__(entity_id='confStatus', label='Confidentiality - status', description='Information about the confidentiality status of the object to which this attribute is attached.', concept_id='confStatus', identifier='confStatus', entity_range='xsd:string') - # self.data = { - # "id": "urn:ngsi-ld:AttributeProperty:confStatus", - # "type": "AttributeProperty", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "rdfs:label": { - # "type": "Property", - # "value": { - # "en": "Confidentiality - status", - # } - # }, - # "dct:description": { - # "type": "Property", - # "value": { - # "en": "Information about the confidentiality status of the object " - # "to which this attribute is attached.", - # } - # }, - # "concept": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:Concept:confStatus" - # }, - # "dct:identifier": { - # "type": "Property", - # "value": "confStatus" - # }, - # "rdfs:range": { - # "type": "Property", - # "value": "xsd:string" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "qb": "http://purl.org/linked-data/cube#", - # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - # } - # } def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value diff --git a/sdmx2jsonld/sdmxattributes/decimals.py b/sdmx2jsonld/sdmxattributes/decimals.py index 63fec71..a1828a5 100644 --- a/sdmx2jsonld/sdmxattributes/decimals.py +++ b/sdmx2jsonld/sdmxattributes/decimals.py @@ -29,50 +29,9 @@ def __init__(self): # rdfs:label "Decimals"@en ; # rdfs:comment """The number of digits of an observation to the right of a decimal point."""@en ; # rdfs:isDefinedBy . - # super().__init__(entity='AttributeProperty') super().__init__(entity_id='decimals', label='Decimals', description='The number of digits of an observation to the right of a decimal point.', concept_id='decimals', identifier='decimals', entity_range='xsd:integer') - # self.data = { - # "id": "urn:ngsi-ld:AttributeProperty:decimals", - # "type": "AttributeProperty", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "rdfs:label": { - # "type": "Property", - # "value": { - # "en": "Decimals", - # } - # }, - # "dct:description": { - # "type": "Property", - # "value": { - # "en": "The number of digits of an observation to the right of a decimal point.", - # } - # }, - # "concept": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:Concept:decimals" - # }, - # "dct:identifier": { - # "type": "Property", - # "value": "decimals" - # }, - # "rdfs:range": { - # "type": "Property", - # "value": "xsd:integer" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "qb": "http://purl.org/linked-data/cube#", - # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - # } - # } diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index fd41efd..6ff2559 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -54,53 +54,12 @@ def __init__(self): # rdfs:label "Observation Status"@en ; # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; # rdfs:isDefinedBy . - # super().__init__(entity='AttributeProperty') super().__init__(entity_id='obsStatus', label='Observation Status', description='Information on the quality of a value or an unusual or missing value.', concept_id='obsStatus', identifier='obsStatus', entity_range='xsd:string') - # self.data = { - # "id": "urn:ngsi-ld:AttributeProperty:obsStatus", - # "type": "AttributeProperty", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "rdfs:label": { - # "type": "Property", - # "value": { - # "en": "Observation Status", - # } - # }, - # "dct:description": { - # "type": "Property", - # "value": { - # "en": "Information on the quality of a value or an unusual or missing value.", - # } - # }, - # "concept": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:Concept:obsStatus" - # }, - # "dct:identifier": { - # "type": "Property", - # "value": "obsStatus" - # }, - # "rdfs:range": { - # "type": "Property", - # "value": "xsd:string" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "qb": "http://purl.org/linked-data/cube#", - # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - # } - # } def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value diff --git a/sdmx2jsonld/sdmxattributes/timeFormat.py b/sdmx2jsonld/sdmxattributes/timeFormat.py index 19b3e39..9813a9e 100644 --- a/sdmx2jsonld/sdmxattributes/timeFormat.py +++ b/sdmx2jsonld/sdmxattributes/timeFormat.py @@ -29,50 +29,9 @@ def __init__(self): # rdfs:label "Time Format"@en ; # rdfs:comment """Technical format in which time is represented for the measured phenomenon."""@en ; # rdfs:isDefinedBy . - # super().__init__(entity='AttributeProperty') super().__init__(entity_id='timeFormat', label='Time Format', description='Technical format in which time is represented for the measured phenomenon.', concept_id='timeFormat', identifier='timeFormat', entity_range='xsd:string') - # self.data = { - # "id": "urn:ngsi-ld:AttributeProperty:timeFormat", - # "type": "AttributeProperty", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "rdfs:label": { - # "type": "Property", - # "value": { - # "en": "Time Format", - # } - # }, - # "dct:description": { - # "type": "Property", - # "value": { - # "en": "Technical format in which time is represented for the measured phenomenon.", - # } - # }, - # "concept": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:Concept:timeFormat" - # }, - # "dct:identifier": { - # "type": "Property", - # "value": "timeFormat" - # }, - # "rdfs:range": { - # "type": "Property", - # "value": "xsd:string" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "qb": "http://purl.org/linked-data/cube#", - # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - # } - # } diff --git a/sdmx2jsonld/sdmxattributes/timePerCollect.py b/sdmx2jsonld/sdmxattributes/timePerCollect.py index a56eada..80cefc0 100644 --- a/sdmx2jsonld/sdmxattributes/timePerCollect.py +++ b/sdmx2jsonld/sdmxattributes/timePerCollect.py @@ -31,7 +31,6 @@ def __init__(self): # (such as middle, average or end of period) to compile the indicator # for the target reference period."""@en ; # rdfs:isDefinedBy . - # super().__init__(entity='AttributeProperty') super().__init__(entity_id='timePerCollect', label='Time Period - collection', description='Dates or periods during which the observations have been collected ' @@ -40,45 +39,3 @@ def __init__(self): concept_id='timePerCollect', identifier='timePerCollect', entity_range='xsd:string') - # self.data = { - # "id": "urn:ngsi-ld:AttributeProperty:timePerCollect", - # "type": "AttributeProperty", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "rdfs:label": { - # "type": "Property", - # "value": { - # "en": "Time Period - collection", - # } - # }, - # "dct:description": { - # "type": "Property", - # "value": { - # "en": "Dates or periods during which the observations have been collected " - # "(such as middle, average or end of period) to compile the indicator " - # "for the target reference period.", - # } - # }, - # "concept": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:Concept:timePerCollect" - # }, - # "dct:identifier": { - # "type": "Property", - # "value": "timePerCollect" - # }, - # "rdfs:range": { - # "type": "Property", - # "value": "xsd:string" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "qb": "http://purl.org/linked-data/cube#", - # "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py index 906c6bd..d27d733 100644 --- a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -33,37 +33,6 @@ def __init__(self): # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS"; # skos:broader sdmx-concept:conf; # skos:inScheme sdmx-concept:cog . - # super().__init__(entity='Concept') super().__init__(entity_id='confStatus', label='Confidentiality - status', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS') - - # self.data = { - # "id": "urn:ngsi-ld:Concept:confStatus", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Confidentiality - status" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/decimals.py b/sdmx2jsonld/sdmxconcepts/decimals.py index eac4d8e..13770f0 100644 --- a/sdmx2jsonld/sdmxconcepts/decimals.py +++ b/sdmx2jsonld/sdmxconcepts/decimals.py @@ -30,36 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS"; # skos:inScheme sdmx-concept:cog . - # super().__init__(entity='Concept') super().__init__(entity_id='decimals', label='Decimals', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS') - # self.data = { - # "id": "urn:ngsi-ld:Concept:decimals", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Decimals" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index 56e530c..891e362 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -30,37 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ"; # skos:inScheme sdmx-concept:cog. - #super().__init__(entity='Concept') super().__init__(entity_id='freq', label='Frequency', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ') - - # self.data = { - # "id": "urn:ngsi-ld:Concept:freq", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Frequency" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py index b872cad..5639632 100644 --- a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -30,37 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; # skos:inScheme sdmx-concept:cog . - # super().__init__(entity='Concept') super().__init__(entity_id='obsStatus', label='Observation Status', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS') - - # self.data = { - # "id": "urn:ngsi-ld:Concept:obsStatus", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Observation Status" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index 6706aa8..ac77804 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -30,38 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA"; # skos:inScheme sdmx-concept:cog . - - # super().__init__(entity='Concept') super().__init__(entity_id='refArea', label='Reference Area', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA') - - # self.data = { - # "id": "urn:ngsi-ld:Concept:refArea", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Reference Area" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py index 09f9082..5e66a4a 100644 --- a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py +++ b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py @@ -34,37 +34,6 @@ def __init__(self): # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT"; # skos:broader sdmx-concept:timePeriod; # skos:inScheme sdmx-concept:cog . - # super().__init__(entity='Concept') super().__init__(entity_id='timePerCollect', label='Time Period - collection', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT') - # self.data = { - # "id": "urn:ngsi-ld:Concept:timePerCollect", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Time Period - collection" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": - # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py index f91def1..73ed042 100644 --- a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py @@ -30,36 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT"; # skos:inScheme sdmx-concept:cog . - # super().__init__(entity='Concept') super().__init__(entity_id='timeFormat', label='Time Format', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT') - # self.data = { - # "id": "urn:ngsi-ld:Concept:timeFormat", - # "type": "Concept", - # "dct:language": { - # "type": "Property", - # "value": ["en"] - # }, - # "skos:inScheme": { - # "type": "Relationship", - # "object": "urn:ngsi-ld:ConceptSchema:cog" - # }, - # "skos:prefLabel": { - # "type": "Property", - # "value": { - # "en": "Time Format" - # } - # }, - # "skos:notation": { - # "type": "Property", - # "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT" - # }, - # "@context": { - # "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - # "dcat": "http://www.w3.org/ns/dcat#", - # "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - # "dct": "http://purl.org/dc/terms/", - # "skos": "http://www.w3.org/2004/02/skos/core#" - # } - # } diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index 52e1839..9c00df6 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -30,37 +30,6 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; # skos:inScheme sdmx-concept:cog. - #super().__init__(entity='Concept') super().__init__(entity_id='timePeriod', label='Time Period', notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD') - - self.data = { - "id": "urn:ngsi-ld:Concept:timePeriod", - "type": "Concept", - "dct:language": { - "type": "Property", - "value": ["en"] - }, - "skos:inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "skos:prefLabel": { - "type": "Property", - "value": { - "en": "Time Period" - } - }, - "skos:notation": { - "type": "Property", - "value": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD" - }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } - } From 9b7ecfea0673d493cda8c3a6073661a5bbfff02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 09:09:46 +0200 Subject: [PATCH 39/74] Update sdmx-attributes and sdmx-concepts --- sdmx2jsonld/sdmxattributes/compilingorg.py | 37 ++++++++++++++++++ sdmx2jsonld/sdmxattributes/currency.py | 37 ++++++++++++++++++ sdmx2jsonld/sdmxattributes/dataComp.py | 39 +++++++++++++++++++ sdmx2jsonld/sdmxattributes/dissorg.py | 37 ++++++++++++++++++ sdmx2jsonld/sdmxattributes/unitmult.py | 39 +++++++++++++++++++ .../sdmxconcepts/compilingorgconcept.py | 37 ++++++++++++++++++ sdmx2jsonld/sdmxconcepts/currencyconcept.py | 37 ++++++++++++++++++ sdmx2jsonld/sdmxconcepts/datacompconcept.py | 38 ++++++++++++++++++ sdmx2jsonld/sdmxconcepts/dissorgconcept.py | 36 +++++++++++++++++ sdmx2jsonld/sdmxconcepts/unitmultconcept.py | 37 ++++++++++++++++++ sdmx2jsonld/transform/dataset.py | 28 +++++++++++-- 11 files changed, 399 insertions(+), 3 deletions(-) create mode 100644 sdmx2jsonld/sdmxattributes/compilingorg.py create mode 100644 sdmx2jsonld/sdmxattributes/currency.py create mode 100644 sdmx2jsonld/sdmxattributes/dataComp.py create mode 100644 sdmx2jsonld/sdmxattributes/dissorg.py create mode 100644 sdmx2jsonld/sdmxattributes/unitmult.py create mode 100644 sdmx2jsonld/sdmxconcepts/compilingorgconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/currencyconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/datacompconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/dissorgconcept.py create mode 100644 sdmx2jsonld/sdmxconcepts/unitmultconcept.py diff --git a/sdmx2jsonld/sdmxattributes/compilingorg.py b/sdmx2jsonld/sdmxattributes/compilingorg.py new file mode 100644 index 0000000..8719865 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/compilingorg.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class CompilingOrg(SDMXAttribute): + def __init__(self): + # sdmx-attribute:compilingOrg a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:compilingOrg ; + # rdfs:label "Compiling agency"@en ; + # rdfs:comment """The organisation compiling the data being reported."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='compilingOrg', + label='Compiling agency', + description='The organisation compiling the data being reported.', + concept_id='compilingOrg', + identifier='compilingOrg', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxattributes/currency.py b/sdmx2jsonld/sdmxattributes/currency.py new file mode 100644 index 0000000..73b65bf --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/currency.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class Currency(SDMXAttribute): + def __init__(self): + # sdmx-attribute:currency a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:currency ; + # rdfs:label "Currency"@en ; + # rdfs:comment """Monetary denomination of the object being measured."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='currency', + label='Currency', + description='Monetary denomination of the object being measured.', + concept_id='currency', + identifier='currency', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxattributes/dataComp.py b/sdmx2jsonld/sdmxattributes/dataComp.py new file mode 100644 index 0000000..e5cf8b3 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/dataComp.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class DataComp(SDMXAttribute): + def __init__(self): + # sdmx-attribute:dataComp a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:dataComp ; + # rdfs:label "Data Compilation"@en ; + # rdfs:comment """Operations performed on data to derive new information according + # to a given set of rules."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='dataComp', + label='Data Compilation', + description='Operations performed on data to derive new information according to a ' + 'given set of rules.', + concept_id='dataComp', + identifier='dataComp', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxattributes/dissorg.py b/sdmx2jsonld/sdmxattributes/dissorg.py new file mode 100644 index 0000000..d7ea500 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/dissorg.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class DissOrg(SDMXAttribute): + def __init__(self): + # sdmx-attribute:dissOrg a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:dissOrg ; + # rdfs:label "Data Dissemination Agency"@en ; + # rdfs:comment """The organisation disseminating the data."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='dissOrg', + label='Data Dissemination Agency', + description='The organisation disseminating the data.', + concept_id='dissOrg', + identifier='dissOrg', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxattributes/unitmult.py b/sdmx2jsonld/sdmxattributes/unitmult.py new file mode 100644 index 0000000..913baa5 --- /dev/null +++ b/sdmx2jsonld/sdmxattributes/unitmult.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxattributes.sdmxattribute import SDMXAttribute + + +class UnitMult(SDMXAttribute): + def __init__(self): + # sdmx-attribute:unitMult a qb:AttributeProperty, rdf:Property ; + # qb:concept sdmx-concept:unitMult ; + # rdfs:label "Unit Multiplier"@en ; + # rdfs:comment """Exponent in base 10 specified so that multiplying the observation + # numeric values by 10^UNIT_MULT gives a value expressed in the UNIT."""@en ; + # rdfs:isDefinedBy . + super().__init__(entity_id='unitMult', + label='Unit Multiplier', + description='Exponent in base 10 specified so that multiplying the observation numeric ' + 'values by 10^UNIT_MULT gives a value expressed in the UNIT.', + concept_id='unitMult', + identifier='unitMult', + entity_range='xsd:integer') diff --git a/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py b/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py new file mode 100644 index 0000000..3f1168d --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class CompilingOrgConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:compilingOrg a sdmx:Concept, skos:Concept ; + # rdfs:label "Compiling agency"@en ; + # rdfs:comment """The organisation compiling the data being reported."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].COMPILING_ORG"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='compilingOrg', + label='Compiling agency', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=' + 'SDMX:CROSS_DOMAIN_CONCEPTS[1.0].COMPILING_ORG') diff --git a/sdmx2jsonld/sdmxconcepts/currencyconcept.py b/sdmx2jsonld/sdmxconcepts/currencyconcept.py new file mode 100644 index 0000000..3cba4f7 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/currencyconcept.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class CurrencyConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:currency a sdmx:Concept, skos:Concept ; + # rdfs:label "Currency"@en ; + # rdfs:comment """Monetary denomination of the object being measured."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CURRENCY"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='currency', + label='Currency', + notation= + 'urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CURRENCY') diff --git a/sdmx2jsonld/sdmxconcepts/datacompconcept.py b/sdmx2jsonld/sdmxconcepts/datacompconcept.py new file mode 100644 index 0000000..02c189e --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/datacompconcept.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class DataCompConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:dataComp a sdmx:Concept, skos:Concept ; + # rdfs:label "Data Compilation"@en ; + # rdfs:comment """Operations performed on data to derive new information according + # to a given set of rules."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='dataComp', + label='Data Compilation', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=' + 'SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP') diff --git a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py new file mode 100644 index 0000000..9b21c63 --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class DissOrgConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:dissOrg a sdmx:Concept, skos:Concept ; + # rdfs:label "Data Dissemination Agency"@en ; + # rdfs:comment """The organisation disseminating the data."""@en ; + # rdfs:isDefinedBy ; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='dissOrg', + label='Data Dissemination Agency', + notation= + 'urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG') diff --git a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py new file mode 100644 index 0000000..b89c25d --- /dev/null +++ b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.sdmxconcepts.sdmxconcept import SDMXConcept + + +class UnitMultConcept(SDMXConcept): + def __init__(self): + # sdmx-concept:unitMult a sdmx:Concept, skos:Concept ; + # rdfs:label "Unit Multiplier"@en ; + # rdfs:comment """Exponent in base 10 specified so that multiplying the observation numeric values + # by 10^UNIT_MULT gives a value expressed in the UNIT."""@en ; + # rdfs:isDefinedBy ; + # skos:notation + # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT"; + # skos:inScheme sdmx-concept:cog . + super().__init__(entity_id='unitMult', + label='Unit Multiplier', + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT') diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 3edff8a..1112a7e 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -35,6 +35,11 @@ from sdmx2jsonld.sdmxattributes.timePerCollect import TimePerCollect from sdmx2jsonld.sdmxattributes.decimals import Decimals from sdmx2jsonld.sdmxattributes.title import Title +from sdmx2jsonld.sdmxattributes.unitmult import UnitMult +from sdmx2jsonld.sdmxattributes.compilingorg import CompilingOrg +from sdmx2jsonld.sdmxattributes.dataComp import DataComp +from sdmx2jsonld.sdmxattributes.currency import Currency +from sdmx2jsonld.sdmxattributes.dissorg import DissOrg from sdmx2jsonld.sdmxconcepts.freqconcept import FreqConcept from sdmx2jsonld.sdmxconcepts.cogconceptschema import CogConceptSchema @@ -46,6 +51,11 @@ from sdmx2jsonld.sdmxconcepts.timePerCollectConcept import TimePerCollectConcept from sdmx2jsonld.sdmxconcepts.decimals import DecimalsConcept from sdmx2jsonld.sdmxconcepts.titleConcept import TitleConcept +from sdmx2jsonld.sdmxconcepts.unitmultconcept import UnitMultConcept +from sdmx2jsonld.sdmxconcepts.compilingorgconcept import CompilingOrgConcept +from sdmx2jsonld.sdmxconcepts.datacompconcept import DataCompConcept +from sdmx2jsonld.sdmxconcepts.currencyconcept import CurrencyConcept +from sdmx2jsonld.sdmxconcepts.dissorgconcept import DissOrgConcept logger = getLogger() @@ -133,7 +143,12 @@ def __init__(self): "timeFormat": TimeFormat(), "timePerCollect": TimePerCollect(), "decimals": Decimals(), - "title": Title() + "title": Title(), + "unitMult": UnitMult(), + "compilingOrg": CompilingOrg(), + "dataComp": DataComp(), + "dissOrg": DissOrg(), + "currency": Currency() } self.sdmx_components = { @@ -150,7 +165,12 @@ def __init__(self): "timeFormat": TimeFormatConcept(), "timePerCollect": TimePerCollectConcept(), "decimals": DecimalsConcept(), - "title": TitleConcept() + "title": TitleConcept(), + "unitMult": UnitMultConcept(), + "compilingOrg": CompilingOrgConcept(), + "dataComp": DataCompConcept(), + "dissOrg": DissOrgConcept(), + "currency": CurrencyConcept() } self.sdmx_concept_schemas = CogConceptSchema() @@ -164,7 +184,9 @@ def add_components(self, context, component): # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix list_special_components = ['freq', 'refArea', 'timePeriod', 'obsStatus', 'confStatus', 'timeFormat', - 'timePerCollect', 'decimals', 'title'] + 'timePerCollect', 'decimals', 'title', + 'unitMult', 'compilingOrg', 'dataComp', + 'currency', 'dissOrg'] type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 From 2820f16efa5bfb6c644ce93e391ea010fa9bfacb Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 20 Jun 2023 09:35:52 +0200 Subject: [PATCH 40/74] Updated data generation and integrated CB --- api/server.py | 16 +++++------ sdmx2jsonld/common/commonclass.py | 1 + sdmx2jsonld/transform/parser.py | 35 ++++++++++++++++------- sdmx2jsonld/transform/transformer.py | 1 + tests/common/test_commonclass.py | 6 ++-- tests/sdmxattributes/common/config.json | 10 +++++++ tests/sdmxattributes/test_examples.py | 38 +++++++++++++++++++++++++ 7 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 tests/sdmxattributes/common/config.json create mode 100644 tests/sdmxattributes/test_examples.py diff --git a/api/server.py b/api/server.py index 62d468c..29bc00b 100644 --- a/api/server.py +++ b/api/server.py @@ -35,6 +35,7 @@ from json import load, loads from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken from io import StringIO +from ngsild.ngsild_connector import NGSILDConnector initial_uptime = datetime.now() logger = getLogger(__name__) @@ -148,7 +149,7 @@ async def parse(request: Request, file: UploadFile, response: Response): # Send the data to a FIWARE Context Broker instance headers = { 'Content-Type': 'application/ld+json', - 'Accept': 'application/ld+json' + # 'Accept': 'application/ld+json' } url = get_url() @@ -156,11 +157,10 @@ async def parse(request: Request, file: UploadFile, response: Response): try: request.app.logger.debug(f'Sending data:\n{json_object}') - - r = post(url=url, headers=headers, data=json_object, timeout=5) - - resp = loads(r.text) - response.status_code = r.status_code + cb = NGSILDConnector() + resp = cb.send_data_array(json_object) + # resp = loads(r.text) + # response.status_code = r.status_code except exceptions.Timeout as err: request.app.logger.error('Timeout requesting FIWARE Context Broker') raise HTTPException(status_code=status.HTTP_408_REQUEST_TIMEOUT, detail=str(err)) @@ -174,9 +174,9 @@ async def parse(request: Request, file: UploadFile, response: Response): except KeyboardInterrupt: request.app.logger.warning('Server interrupted by user') raise - except Exception: + except Exception as e: message = "Unknown error sending data to the Context Broker" - request.app.logger.error(message) + request.app.logger.error(e.message) raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(message)) else: request.app.logger.info(f'Content sent to the Context Broker') diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index 1e80c7b..a30e31d 100644 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -91,6 +91,7 @@ def generate_id(self, value, entity=None, update_id=False): else: return aux, new_aux + def __generate_property__(self, key, value): result = { key: { diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index ddd5b06..c7bdd0e 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -88,21 +88,23 @@ def parsing_file(self, content: TextIOWrapper, out: bool): catalogue = transform.get_catalogue() pprint(catalogue) - self.__check_pprint__(transform.get_dataset()) + ds = transform.get_dataset() + if ds is not None: + self.__check_pprint__(transform.get_dataset()) [pprint(x.get()) for x in transform.get_dimensions()] [pprint(x.get()) for x in transform.get_attributes()] [pprint(x.get()) for x in transform.get_concept_schemas()] [pprint(x.get()) for x in transform.get_concept_lists()] - observations = transform.get_observation() + observations = transform.entity_type.get_observation() if len(observations) != 0: [pprint(x.get()) for x in observations] # If we have several observations, we need to generate the DCAT-AP:Distribution class distribution = Distribution() - distribution.generate_data(catalogue=catalogue) + distribution.generate_data(catalogue=transform.entity_type.catalogue) - pprint(distribution) + pprint(distribution.get()) def parsing_string(self, content: StringIO): transform = TreeToJson() @@ -115,24 +117,35 @@ def parsing_string(self, content: StringIO): # Serializing json payload result = list() + catalogue = transform.get_catalogue() result.append(catalogue) - result.append(transform.get_dataset()) - [result.append(x.get()) for x in transform.get_dimensions()] + + ds = transform.get_dataset() + if ds is not None: + result.append(ds) + + dimensions = transform.get_dimensions() + [result.append(x.get()) for x in dimensions] + + attr = transform.get_attributes() [result.append(x.get()) for x in transform.get_attributes()] + [result.append(x.get()) for x in transform.get_concept_schemas()] + [result.append(x.get()) for x in transform.get_concept_lists()] - [result.append(x.get()) for x in transform.get_observation()] + # jicg - [result.append(x.get()) for x in transform.get_observation()] - observations = transform.get_observation() + observations = transform.entity_type.get_observation() + # jicg - observations = transform.get_observation() if len(observations) != 0: [result.append(x.get()) for x in observations] # If we have several observations, we need to generate the DCAT-AP:Distribution class distribution = Distribution() - distribution.generate_data(catalogue=catalogue) - - result.append(distribution) + # jicg. distribution.generate_data(catalogue=catalogue) + distribution.generate_data(catalogue=transform.entity_type.catalogue) + result.append(distribution.get()) json_object = dumps(result, indent=4, ensure_ascii=False) diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index cc3481a..00c3062 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -123,6 +123,7 @@ def get_observation(self): def get_dataset(self): if self.entity_type.dataset.data['id'] != '': return self.entity_type.get_dataset() + return None def get_dimensions(self): return self.entity_type.get_dimensions() diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py index 1b5e8a5..af5590c 100644 --- a/tests/common/test_commonclass.py +++ b/tests/common/test_commonclass.py @@ -31,7 +31,7 @@ def setUp(self) -> None: def test_instance_class(self): cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id("https://string-to-parse-ur/entity_id") + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) assert (urnid == "urn:ngsi-ld:test.common.entity:entity_id") # urnid = cclass.generate_id("") # print(urnid) @@ -42,7 +42,7 @@ def test_save(self): "alternateName": "https://smartdatamodels.org/alternateName", "status": "ngsi-ld:status"} cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id("https://string-to-parse-ur/entity_id") + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) assert(urnid == "urn:ngsi-ld:test.common.entity:entity_id") cclass.add_context(context, context_map) @@ -55,3 +55,5 @@ def test_save(self): data = json.load(f) assert(data['id'] == urnid) assert(data['@context'] == context['@context']) + + # TODO - Add tests with cclass.generate_id using update_id with a False value \ No newline at end of file diff --git a/tests/sdmxattributes/common/config.json b/tests/sdmxattributes/common/config.json new file mode 100644 index 0000000..a06283f --- /dev/null +++ b/tests/sdmxattributes/common/config.json @@ -0,0 +1,10 @@ +{ + "broker": "http://127.0.0.1:1026", + "logger": { + "path": "./logs/access.log", + "level": "debug", + "rotation": "20 days", + "retention": "1 months", + "format": "{level: <8} {time:YYYY-MM-DD HH:mm:ss.SSS} request id: {extra[request_id]} - {name}:{function} - {message}" + } +} diff --git a/tests/sdmxattributes/test_examples.py b/tests/sdmxattributes/test_examples.py new file mode 100644 index 0000000..ed34128 --- /dev/null +++ b/tests/sdmxattributes/test_examples.py @@ -0,0 +1,38 @@ +from unittest import TestCase +from sdmx2jsonld.transform.parser import Parser +from io import StringIO +import os, shutil + +class TestCommonClass(TestCase): + def setUp(self) -> None: + pass + + def clean_output_dir(self) -> None: + for a in os.listdir("output"): + file_path = os.path.join("output", a) + try: + if os.path.isfile(file_path): + os.unlink(file_path) + except Exception as e: + print("----------------------------") + print(e.message) + print("----------------------------") + + def test_file_1(self): + for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: + print(f"Parsing: {a}") + parser = Parser() + with open(f"../../examples/{a}", "r") as rf: + rdf_data = rf.read() + r = parser.parsing(content=StringIO(rdf_data), out=False) + + def test_file_2(self): + for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: + self.clean_output_dir() + parser = Parser() + r = parser.parsing(content=open(f"../../examples/{a}", "r"), out=True) + + def test_file_3(self): + for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: + parser = Parser() + r = parser.parsing(content=open(f"../../examples/{a}", "r"), out=False) From ded7aba25d83618fade5bf16b16c5a682b5ef410 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 20 Jun 2023 09:37:29 +0200 Subject: [PATCH 41/74] Integrated cb-connector --- ngsild/ngsild_connector.py | 42 +++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index 322f30d..fcf40ca 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -1,28 +1,8 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -## -# Copyright 2022 FIWARE Foundation, e.V. -# -# This file is part of IoTAgent-SDMX (RDF Turtle) -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## from pathlib import Path import json from requests import post, exceptions + class NGSILDConnector: def __init__(self, path=None): if path == None: @@ -39,11 +19,27 @@ def get_url(self): return url def send_data_array(self, json_object): + return_info = [] d = json.loads(json_object) d = d if type(d) is list else [d] for elem in d: - rc, r = c.send_data(json.dumps(elem)) + try: + rc, r = self.send_data(json.dumps(elem)) + return_info.append({"id": elem['id'], + "status_code": rc, + "reason": r}) + except TypeError as e: + return_info.append({"id": "UNK", + "status_code": 500, + "reason": e.args[0]}) + except Exception as e: + return_info.append({"id": "UNK", + "status_code": 500, + "reason": e.message}) + + return return_info + def send_data(self, json_object): # Send the data to a FIWARE Context Broker instance @@ -64,7 +60,7 @@ def send_data(self, json_object): print("LOCATION: ", r.headers['Location']) # Let exceptions raise.... They can be controlled somewhere else. - return response_status_code, resp + return response_status_code, r.reason from sdmx2jsonld.transform.parser import Parser From 8f04f8c20745c902a1285ca7c1115831c5b2c2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 10:13:26 +0200 Subject: [PATCH 42/74] Correct sdmx-measure, it is not a unitStatMeasure --- sdmx2jsonld/transform/dataset.py | 38 ++++++++++++++++++----------- sdmx2jsonld/transform/entitytype.py | 11 ++------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 1112a7e..181ba38 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -64,6 +64,14 @@ class Dataset(CommonClass): def __init__(self): super().__init__(entity='Dataset') + # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment + # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix + self.list_special_components = ['freq', 'refArea', 'timePeriod', + 'obsStatus', 'confStatus', 'timeFormat', + 'timePerCollect', 'decimals', 'title', + 'unitMult', 'compilingOrg', 'dataComp', + 'currency', 'dissOrg'] + self.data = { "id": str(), "type": "Dataset", @@ -180,17 +188,22 @@ def add_components(self, context, component): # qb:attribute, qb:dimension, or qb:measure list_components = ['qb:attribute', 'qb:dimension', 'qb:measure'] - # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment - # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix - list_special_components = ['freq', 'refArea', 'timePeriod', - 'obsStatus', 'confStatus', 'timeFormat', - 'timePerCollect', 'decimals', 'title', - 'unitMult', 'compilingOrg', 'dataComp', - 'currency', 'dissOrg'] - type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 + if type_component == "qb:measure": + logger.info(f'The qb:measure "{component[position][0]}" is not manage in statDCAT-AP') + new_component, new_concept, new_concept_schema = None, None, None + else: + new_component, new_concept, new_concept_schema = self.manage_components(type_component=type_component, + component=component, + position=position, + context=context) + + return new_component, new_concept, new_concept_schema + + def manage_components(self, type_component, component, position, context): + new_component, new_concept, new_concept_schema = None, None, None try: entity = self.components[type_component]['entity'] name, new_id = self.generate_id(entity=entity, value=component[position][0], update_id=False) @@ -200,7 +213,7 @@ def add_components(self, context, component): if new_id in self.components[type_component]['value'][key]['object']: logger.warning( f"The component {new_id} is duplicated and already defined in the {self.data['id']}") - elif name in list_special_components: + elif name in self.list_special_components: # We need to create manually the description of these dimensions, concepts, and conceptschemas logger.warning( f"The component {name} is defined probably outside of the file, " @@ -208,12 +221,9 @@ def add_components(self, context, component): self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] - new_component = self.sdmx_components[entity] - new_dimension = new_component[name] + new_component = self.sdmx_components[entity][name] new_concept = self.sdmx_concepts[name] new_concept_schema = self.sdmx_concept_schemas - - return new_dimension, new_concept, new_concept_schema else: self.components[type_component]['value'][key]['object'].append(new_id) self.data = self.data | self.components[type_component]['value'] @@ -231,7 +241,7 @@ def add_components(self, context, component): a.order_context() self.data = a.get_data() - return None, None, None + return new_component, new_concept, new_concept_schema def get(self): return self.data diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index d27fa60..79bb7c7 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -213,15 +213,8 @@ def create_data(self, entity_type, data, title): data_range.add_data(concept_id=data_range_id, data=data) self.conceptLists.append(data_range) self.conceptListsIds[title] = data_range.get_id() - - # for i in range(0, len(self.conceptSchemas)): - # concept_schema = self.conceptSchemas[i].data - # has_top_concept_values = concept_schema['skos:hasTopConcept']['value'] - # - # out = [data_range.data['skos:notation'] - # if x == data_range.data['id'] else x for x in has_top_concept_values] - # - # self.conceptSchemas[i].data['skos:hasTopConcept']['value'] = out + else: + logger.error(f'Entity type "{entity_type}" not processed.') def __get_subject__(self, title): if self.dataset.get()['dct:title'] == title: From fbf113fb4624e262f950d9a8686758ee40dd797a Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 20 Jun 2023 12:17:30 +0200 Subject: [PATCH 43/74] Added docker file --- api/server.py | 6 +-- docker/Dockerfile | 82 ++++++++++++++++---------------------- docker/config.json | 2 +- ngsild/ngsild_connector.py | 8 ++-- 4 files changed, 43 insertions(+), 55 deletions(-) diff --git a/api/server.py b/api/server.py index 29bc00b..be221f1 100644 --- a/api/server.py +++ b/api/server.py @@ -175,9 +175,9 @@ async def parse(request: Request, file: UploadFile, response: Response): request.app.logger.warning('Server interrupted by user') raise except Exception as e: - message = "Unknown error sending data to the Context Broker" - request.app.logger.error(e.message) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(message)) + r = getattr(e, 'message', str(e)) + request.app.logger.error(r) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(r)) else: request.app.logger.info(f'Content sent to the Context Broker') request.app.logger.debug(f'Status Code: {response.status_code}, Response:\n{resp}') diff --git a/docker/Dockerfile b/docker/Dockerfile index 03f69d2..8c9b420 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,63 +1,49 @@ -FROM python:3.9-alpine@sha256:e80214a705236091ee3821a7e512e80bd3337b50a95392a36b9a40b8fc0ea183 as build +FROM alpine/git as git-clone -LABEL "maintainer"="FIWARE Foundation e.V." -LABEL "description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." -LABEL "name"="iotagent-turtle" -LABEL "summary"="IoT Agent for the DataQube representation in RDF Turtle format used in statistical environments" +ARG PROJECT=flopezag +ARG COMPONENT=IoTAgent-Turtle +ARG VERSION=integration-cb-httpserver +# ARG VERSION=development +ARG INSTALLATION_PATH=/opt/$COMPONENT -LABEL "org.opencontainers.image.authors"="fernando.lopez@fiware.org" -LABEL "org.opencontainers.image.documentation"="" -LABEL "org.opencontainers.image.vendor"="FIWARE Foundation e.V." -LABEL "org.opencontainers.image.licenses"="Apache2.0" -LABEL "org.opencontainers.image.title"="iotagent-turtle" -LABEL "org.opencontainers.image.description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." -LABEL "org.opencontainers.image.source"="https://github.com/flopezag/IoTAgent-Turtle" -LABEL "org.opencontainers.image.version"="0.1.0" -LABEL "org.python.version"="python:3.9" +RUN mkdir -p $INSTALLATION_PATH +RUN git clone https://github.com/$PROJECT/$COMPONENT --branch $VERSION $INSTALLATION_PATH -RUN apk update -# RUN apt-get install -y --no-install-recommends \ -# build-essential gcc -WORKDIR /usr/app -RUN python -m venv /usr/app/venv -ENV PATH="/usr/app/venv/bin:$PATH" +## Install PIP Requirements +FROM python:3.11-alpine as pip-requirements -COPY requirements.txt . -RUN pip install --no-cache-dir --upgrade -r requirements.txt +ARG PROJECT=flopezag +ARG COMPONENT=IoTAgent-Turtle +ARG VERSION=integration-cb-httpserver +# ARG VERSION=development +ARG INSTALLATION_PATH=/opt/$COMPONENT -FROM python:3.9-alpine@sha256:e80214a705236091ee3821a7e512e80bd3337b50a95392a36b9a40b8fc0ea183 +RUN mkdir -p $INSTALLATION_PATH +COPY --from=git-clone $INSTALLATION_PATH/requirements.txt /requirements.txt +RUN pip install --root-user-action=ignore --prefix=$INSTALLATION_PATH -r /requirements.txt -ENV PORT=${IOTA_PORT:-5000} -LABEL "maintainer"="FIWARE Foundation e.V." -LABEL "description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." -LABEL "name"="iotagent-turtle" -LABEL "summary"="IoT Agent for the DataQube representation in RDF Turtle format used in statistical environments" +FROM python:3.11-alpine as final -LABEL "org.opencontainers.image.authors"="fernando.lopez@fiware.org" -LABEL "org.opencontainers.image.documentation"="" -LABEL "org.opencontainers.image.vendor"="FIWARE Foundation e.V." -LABEL "org.opencontainers.image.licenses"="Apache2.0" -LABEL "org.opencontainers.image.title"="iotagent-turtle" -LABEL "org.opencontainers.image.description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." -LABEL "org.opencontainers.image.source"="https://github.com/flopezag/IoTAgent-Turtle" -LABEL "org.opencontainers.image.version"="0.1.0" -LABEL "org.python.version"="python:3.9" +ARG PROJECT=flopezag +ARG COMPONENT=IoTAgent-Turtle +ARG VERSION=integration-cb-httpserver +# ARG VERSION=development +ARG INSTALLATION_PATH=/opt/$COMPONENT -RUN addgroup -g 99 python && \ - adduser -S -u 999 -g python python +ENV PORT=${IOTA_PORT:-5000} -RUN mkdir /usr/app && chown python:python /usr/app -WORKDIR /usr/app +RUN mkdir -p $INSTALLATION_PATH -COPY --chown=python:python --from=build /usr/app/venv ./venv -COPY --chown=python:python . . +COPY --from=git-clone $INSTALLATION_PATH $INSTALLATION_PATH +COPY --from=pip-requirements $INSTALLATION_PATH /usr/local -USER 999 +COPY config.json $INSTALLATION_PATH/common/config.json +WORKDIR $INSTALLATION_PATH -ENV PATH="/usr/app/venv/bin:$PATH" -CMD ["python", "agent.py", "server", "--host", "0.0.0.0", "--port", "5000"] -HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl -f https://localhost:${PORT}/version +#CMD ["python", "agent.py", "server", "--host", "0.0.0.0", "--port", $PORT] +# EXPOSE ${PORT} +ENTRYPOINT /usr/local/bin/python agent.py server --host 0.0.0.0 --port ${PORT} -EXPOSE ${PORT} +expose $PORT diff --git a/docker/config.json b/docker/config.json index a8c047b..81bdc44 100644 --- a/docker/config.json +++ b/docker/config.json @@ -1,5 +1,5 @@ { - "broker": "http://orion:1026", + "broker": "http://orion-ld:1026", "logger": { "path": "./logs/access.log", "level": "debug", diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index fcf40ca..04c1d57 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -34,9 +34,11 @@ def send_data_array(self, json_object): "status_code": 500, "reason": e.args[0]}) except Exception as e: - return_info.append({"id": "UNK", - "status_code": 500, - "reason": e.message}) + raise e + # reason = getattr(e, 'message', str(e)) + # return_info.append({"id": "UNK", + # "status_code": 500, + # "reason": reason}) return return_info From 2544f6738cfa55510da311923127a0f9e5481693 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 20 Jun 2023 13:01:42 +0200 Subject: [PATCH 44/74] Added readme.md and solved some bugs --- docker/Readme.md | 81 ++++++++++++++++++++++++++++++++++++++ docker/docker-compose.yaml | 34 ++++++++++++++++ docker/docker-compose.yml | 81 -------------------------------------- 3 files changed, 115 insertions(+), 81 deletions(-) create mode 100644 docker/Readme.md create mode 100644 docker/docker-compose.yaml delete mode 100644 docker/docker-compose.yml diff --git a/docker/Readme.md b/docker/Readme.md new file mode 100644 index 0000000..7d81a43 --- /dev/null +++ b/docker/Readme.md @@ -0,0 +1,81 @@ +# Docker for server + +## build +The docker is easyly build using the following command: + +``` + docker build . -t iotagent-turtle +``` + +Everything will install in /proc/IotAgent-turtle inside the container. The default config file will be the following one: + +``` +{ + "broker": "http://orion-ld:1026", + "logger": { + "path": "./logs/access.log", + "level": "debug", + "rotation": "20 days", + "retention": "1 months", + "format": "{level: <8} {time:YYYY-MM-DD HH:mm:ss.SSS} request id: {extra[request_id]} - {name}:{function} - {message}" + } +} +``` + +## Use +Let's suppose the image is named **io** and we want to expose port 5000 in order to work. We can also image that we want to remove the container when it finishes, we can do this way: +``` +docker run --rm -p 5000:5000 --name io iotagent-turtle +``` + +However, we might have Orion-ld somewhere else. So if we cant to connect our context broker somewhere, we can use this command: +``` +docker run --rm -p 5000:5000 --name io --add-host=orion-ld:192.168.1.206 iotagent-turtle +``` + +### Overriding config.json +We could create our own config.json and override the one in the docker by default +``` +docker run --rm -p 5000:5000 --name io -v our-local-config.json:/opt/IoTAgent-turtle/common/config.json iotagent-turtle +``` + +### As docker file + +We can consider writing a docker-compose.yaml file as the following one to start everything (orion-ld, the name of our orion server is named according to our config.json file: + +``` +version: "3.8" +services: + orion-ld: + image: fiware/orion-ld + hostname: orion-ld + container_name: orion-ld + expose: + - 1026 + ports: + - 1026:1026 + depends_on: + - fiware-orion-ld-mongo-db + command: -dbhost fiware-orion-ld-mongo-db -logLevel DEBUG -experimental + + fiware-orion-ld-mongo-db: + image: mongo:5.0 + hostname: mongo-db + networks: + - default + command: --nojournal + volumes: + - /data/docker/orion-ld/mongodb:/data + + iotagent-turtle: + image: iotagent-turtle:latest + hostname: ioagent-turtle + container_name: iotagent-turtle + expose: + - 5000 + ports: + - 5000:5000 + networks: + - default +``` + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..edcc4bb --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,34 @@ +version: "3.8" +services: + orion-ld: + image: fiware/orion-ld + hostname: orion-ld + container_name: orion-ld + expose: + - 1026 + ports: + - 1026:1026 + depends_on: + - fiware-orion-ld-mongo-db + command: -dbhost fiware-orion-ld-mongo-db -logLevel DEBUG -experimental + + fiware-orion-ld-mongo-db: + image: mongo:5.0 + hostname: mongo-db + networks: + - default + command: --nojournal + volumes: + - /data/docker/orion-ld/mongodb:/data + + iotagent-turtle: + image: iotagent-turtle:latest + hostname: ioagent-turtle + container_name: iotagent-turtle + expose: + - 5000 + ports: + - 5000:5000 + networks: + - default + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index be57f38..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,81 +0,0 @@ -# WARNING: Do not deploy this tutorial configuration directly to a production environment -# -# The tutorial docker-compose files have not been written for production deployment and will not -# scale. A proper architecture has been sacrificed to keep the narrative focused on the learning -# goals, they are just used to deploy everything onto a single Docker machine. All FIWARE components -# are running at full debug and extra ports have been exposed to allow for direct calls to services. -# They also contain various obvious security flaws - passwords in plain text, no load balancing, -# no use of HTTPS and so on. -# -# This is all to avoid the need of multiple machines, generating certificates, encrypting secrets -# and so on, purely so that a single docker-compose file can be read as an example to build on, -# not use directly. -# -# When deploying to a production environment, please refer to the Helm Repository -# for FIWARE Components in order to scale up to a proper architecture: -# -# see: https://github.com/FIWARE/helm-charts/ -# -version: "3.5" -services: - iotagent: - image: iotagent-turtle - hostname: iotagent-turtle - container_name: iotagent-turtle - networks: - - default - volumes: - - ./config.json:/usr/app/common/config.json:ro - ports: - - "${IOTAGENT_PORT}:${IOTAGENT_PORT}" # localhost:5000 - command: python agent.py server --host 0.0.0.0 --port ${IOTAGENT_PORT} - healthcheck: - test: curl -f https://localhost:${IOTAGENT_PORT}/version || exit 1 - interval: 5s - timeout: 30s - retries: 3 - - orion: - image: fiware/orion-ld:${ORION_LD_VERSION} - hostname: orion - container_name: fiware-orion - depends_on: - - mongo-db - networks: - - default - ports: - - "${ORION_LD_PORT}:${ORION_LD_PORT}" # localhost:1026 - command: -dbhost mongo-db -logLevel DEBUG -forwarding - healthcheck: - test: curl --fail -s http://orion:${ORION_LD_PORT}/version || exit 1 - interval: 5s - - mongo-db: - image: mongo:${MONGO_DB_VERSION} - hostname: mongo-db - container_name: db-mongo - expose: - - "${MONGO_DB_PORT}" - ports: - - "${MONGO_DB_PORT}:${MONGO_DB_PORT}" # localhost:27017 - networks: - - default - volumes: - - mongo-db:/data/db - - mongo-config:/data/configdb - healthcheck: - test: | - host=`hostname --ip-address || echo '127.0.0.1'`; - mongo --quiet $host/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' && echo 0 || echo 1 - interval: 5s - - -networks: - default: - ipam: - config: - - subnet: 172.18.1.0/24 - -volumes: - mongo-db: ~ - mongo-config: From f7ee268e05af8b4d150f758e3a5a68564455d593 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Tue, 20 Jun 2023 13:29:04 +0200 Subject: [PATCH 45/74] Changed a bit Readme.md and set a more convenient branch to pull when docker is done --- docker/Dockerfile | 9 ++------- docker/Readme.md | 7 ++++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8c9b420..582af3e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,7 @@ FROM alpine/git as git-clone ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle -ARG VERSION=integration-cb-httpserver -# ARG VERSION=development +ARG VERSION=develop ARG INSTALLATION_PATH=/opt/$COMPONENT RUN mkdir -p $INSTALLATION_PATH @@ -16,7 +15,6 @@ FROM python:3.11-alpine as pip-requirements ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle ARG VERSION=integration-cb-httpserver -# ARG VERSION=development ARG INSTALLATION_PATH=/opt/$COMPONENT RUN mkdir -p $INSTALLATION_PATH @@ -28,8 +26,6 @@ FROM python:3.11-alpine as final ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle -ARG VERSION=integration-cb-httpserver -# ARG VERSION=development ARG INSTALLATION_PATH=/opt/$COMPONENT ENV PORT=${IOTA_PORT:-5000} @@ -42,8 +38,7 @@ COPY --from=pip-requirements $INSTALLATION_PATH /usr/local COPY config.json $INSTALLATION_PATH/common/config.json WORKDIR $INSTALLATION_PATH -#CMD ["python", "agent.py", "server", "--host", "0.0.0.0", "--port", $PORT] -# EXPOSE ${PORT} +EXPOSE ${PORT} ENTRYPOINT /usr/local/bin/python agent.py server --host 0.0.0.0 --port ${PORT} expose $PORT diff --git a/docker/Readme.md b/docker/Readme.md index 7d81a43..158c330 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -1,12 +1,17 @@ # Docker for server ## build -The docker is easyly build using the following command: +The docker is easyly build using the following command - By default it will build the docker for the **develop** branch: ``` docker build . -t iotagent-turtle ``` +If we want to build other branch, we could define the branch we want to use this way: +``` + docker build . -t iotagent-turtle --build-arg VERSION=integration-cb-httpserver +``` + Everything will install in /proc/IotAgent-turtle inside the container. The default config file will be the following one: ``` From 51c8d03458fdbdc7bd59eea05510a5c6ae133ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 14:32:19 +0200 Subject: [PATCH 46/74] Resolve the prefix resolution in the DimensionProperty class --- sdmx2jsonld/common/listmanagement.py | 21 +++++++++++++- sdmx2jsonld/exceptions/exceptions.py | 13 +++++++++ sdmx2jsonld/transform/catalogue.py | 13 ++++----- sdmx2jsonld/transform/concept.py | 11 ++++---- sdmx2jsonld/transform/conceptschema.py | 13 +++++---- sdmx2jsonld/transform/dataset.py | 16 +++++------ sdmx2jsonld/transform/entitytype.py | 22 +++++++-------- sdmx2jsonld/transform/property.py | 29 ++++++++++--------- sdmx2jsonld/transform/transformer.py | 2 +- tests/common/test_listmanagement.py | 39 +++++++++++++++++++++++++- 10 files changed, 126 insertions(+), 53 deletions(-) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index 40be8b7..b260400 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -20,6 +20,7 @@ # under the License. ## from logging import getLogger +from sdmx2jsonld.exceptions.exceptions import ClassExtractPrefixError logger = getLogger() @@ -86,5 +87,23 @@ def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): filter(lambda x: filter_key_with_prefix(x, not_allowed_keys, further_process_keys), list(aux.keys()))) new_data = {k: aux[k] for k in new_keys} + corrected_dict = {k.replace(k, extract_prefix(k)): v for k, v in new_data.items()} - return new_data + return corrected_dict + + +def extract_prefix(attribute): + result = None + if attribute is None or len(attribute) == 0: + raise ClassExtractPrefixError(data=attribute, message=f"Unexpected data received: '{attribute}'") + else: + data = attribute.split(':') + + if len(data) == 1: + result = data[0] + elif len(data) == 2: + result = data[1] + else: + raise ClassExtractPrefixError(data=attribute, message=f"Unexpected number of prefixes: '{attribute}'") + + return result diff --git a/sdmx2jsonld/exceptions/exceptions.py b/sdmx2jsonld/exceptions/exceptions.py index 30127bd..83d7416 100644 --- a/sdmx2jsonld/exceptions/exceptions.py +++ b/sdmx2jsonld/exceptions/exceptions.py @@ -80,3 +80,16 @@ class ClassFreqError(ClassSDMXAttributeError): def __init__(self, data, message="Decimals value is not the expected"): super().__init__(data=data, message=message) + + +class ClassExtractPrefixError(ClassSDMXAttributeError): + """Raised when the input value is None or Empty or includes several prefixes""" + """Exception raised for errors in the input data. + + Attributes: + data -- input data which caused the error + message -- explanation of the error + """ + + def __init__(self, data, message="Value is not the expected"): + super().__init__(data=data, message=message) diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index d07307a..bbf5a72 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -29,13 +29,13 @@ logger = getLogger() -class CatalogueDCATAP(CommonClass): +class Catalogue(CommonClass): def __init__(self): - super().__init__(entity='CatalogueDCAT-AP') + super().__init__(entity='Catalogue') self.data = { "id": str(), - "type": "CatalogueDCAT-AP", + "type": "Catalogue", "qb:dataset": { "type": "Relationship", @@ -74,8 +74,7 @@ def __init__(self): }, "@context": [ - "https://raw.githubusercontent.com/SEMICeu/DCAT-AP/master/releases/1.1/dcat-ap_1.1.jsonld", - "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" ] } @@ -91,7 +90,7 @@ def add_dataset(self, dataset_id): hash1 = "%032x" % random_bits # Add the id - self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + hash1 + self.data['id'] = "urn:ngsi-ld:Catalogue:" + hash1 # Add dataset id @@ -107,7 +106,7 @@ def add_data(self, title, dataset_id, data): self.data[key]['value'] = title # Add the id - self.data['id'] = "urn:ngsi-ld:CatalogueDCAT-AP:" + dataset_id + self.data['id'] = "urn:ngsi-ld:Catalogue:" + dataset_id # Add the publisher key = self.get_key(requested_key='dcterms:publisher') diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index 951e9ee..7095522 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -67,6 +67,7 @@ def __init__(self): } self.concept_id = str() + self.keys = {k: k for k in self.data.keys()} def add_data(self, concept_id, data): # TODO: We have to control that data include the indexes that we want to search @@ -129,11 +130,11 @@ def add_data(self, concept_id, data): self.need_add_notation(data=data) # Simplify Context and order keys - a = Context() - a.set_data(data=self.data) - a.new_analysis() - a.order_context() - self.data = a.get_data() + #a = Context() + #a.set_data(data=self.data) + #a.new_analysis() + #a.order_context() + #self.data = a.get_data() def get(self): return self.data diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index e90edcd..8d8865a 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -63,6 +63,9 @@ def __init__(self): "@context": dict() } + self.keys = {k: k for k in self.data.keys()} + + def add_data(self, concept_schema_id, data): # TODO: We have to control that data include the indexes that we want to search # We need to complete the data corresponding to the ConceptSchema: skos:prefLabel @@ -124,11 +127,11 @@ def add_data(self, concept_schema_id, data): } # Simplify Context and order keys - a = Context() - a.set_data(data=self.data) - a.new_analysis() - a.order_context() - self.data = a.get_data() + # a = Context() + # a.set_data(data=self.data) + # a.new_analysis() + # a.order_context() + # self.data = a.get_data() def get(self): return self.data diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 181ba38..c3ba757 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -232,14 +232,14 @@ def manage_components(self, type_component, component, position, context): # Simplify Context and order keys. It is possible that we call add_component before the dataset has been created # therefore we need to add the corresponding context to the dataset - if len(self.data['@context']) == 0: - self.data['@context'] = context['@context'] - - a = Context() - a.set_data(data=self.data) - a.new_analysis() - a.order_context() - self.data = a.get_data() + # if len(self.data['@context']) == 0: + # self.data['@context'] = context['@context'] + # + # a = Context() + # a.set_data(data=self.data) + # a.new_analysis() + # a.order_context() + # self.data = a.get_data() return new_component, new_concept, new_concept_schema diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 79bb7c7..f7741e5 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -25,7 +25,7 @@ from sdmx2jsonld.transform.conceptschema import ConceptSchema from sdmx2jsonld.transform.concept import Concept from sdmx2jsonld.transform.attribute import Attribute -from sdmx2jsonld.transform.catalogue import CatalogueDCATAP +from sdmx2jsonld.transform.catalogue import Catalogue from sdmx2jsonld.transform.observation import Observation from logging import getLogger from datetime import datetime @@ -59,7 +59,7 @@ def __init__(self): self.conceptListsIds = dict() self.context = dict() self.context_mapping = dict() - self.catalogue = CatalogueDCATAP() + self.catalogue = Catalogue() self.observations = list() self.pre = Precedence() @@ -110,9 +110,9 @@ def __find_entity_type__(self, string): return data, string1, is_new def transform(self, string): - if len(self.context) == 0: - raise AssertionError("Context should be passed before to the EntityType Class, " - "call EntityType.set_context() before, {'__file__': this_file}))") + #if len(self.context) == 0: + # raise AssertionError("Context should be passed before to the EntityType Class, " + # "call EntityType.set_context() before, {'__file__': this_file}))") data_type, new_string, is_new = self.__find_entity_type__(string=string) @@ -174,33 +174,33 @@ def create_data(self, entity_type, data, title): self.observations.append(observation) elif entity_type == 'Dataset': identifier = self.parser.obtain_id(title) - self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) + #self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) self.dataset.add_data(title=title, dataset_id=identifier, data=data) # Create the CatalogueDCAT-AP and assign the dataset id self.catalogue.add_dataset(dataset_id=self.dataset.data['id']) elif entity_type == 'Dimension': dimension = Dimension() - dimension.add_context(context=self.context, context_mapping=self.context_mapping) + #dimension.add_context(context=self.context, context_mapping=self.context_mapping) dimension_id = self.parser.obtain_id(title) dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) elif entity_type == 'Attribute': attribute = Attribute() - attribute.add_context(context=self.context, context_mapping=self.context_mapping) + #attribute.add_context(context=self.context, context_mapping=self.context_mapping) attribute_id = self.parser.obtain_id(title) attribute.add_data(attribute_id=attribute_id, data=data) self.attributes.append(attribute) elif entity_type == 'ConceptScheme': concept_schema = ConceptSchema() - concept_schema.add_context(context=self.context, context_mapping=self.context_mapping) + #concept_schema.add_context(context=self.context, context_mapping=self.context_mapping) concept_schema_id = self.parser.obtain_id(title) concept_schema.add_data(concept_schema_id=concept_schema_id, data=data) self.conceptSchemas.append(concept_schema) elif entity_type == 'Class': # We need the Concept because each of the Range description is of the type Concept concept_list = Concept() - concept_list.add_context(context=self.context, context_mapping=self.context_mapping) + #concept_list.add_context(context=self.context, context_mapping=self.context_mapping) concept_list_id = self.parser.obtain_id(title) concept_list.add_data(concept_id=concept_list_id, data=data) self.conceptLists.append(concept_list) @@ -208,7 +208,7 @@ def create_data(self, entity_type, data, title): elif entity_type == 'Range': # TODO: Range is associated to a Concept and identified properly in the ConceptSchema data_range = Concept() - data_range.add_context(context=self.context, context_mapping=self.context_mapping) + #data_range.add_context(context=self.context, context_mapping=self.context_mapping) data_range_id = self.parser.obtain_id(title) data_range.add_data(concept_id=data_range_id, data=data) self.conceptLists.append(data_range) diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index 94e8802..4d00767 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -35,7 +35,7 @@ def __init__(self, entity): self.data = { "id": str(), "type": "", - "dct:language": { + "language": { "type": "Property", "value": list() }, @@ -44,26 +44,27 @@ def __init__(self, entity): # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker ################################################# - # "rdfs:label": { + # "label": { # "type": "LanguageProperty", # "LanguageMap": dict() # }, ################################################# - "rdfs:label": { + "label": { "type": "Property", "value": dict() }, - - "qb:codeList": { + "codeList": { "type": "Relationship", "object": str() }, - "qb:concept": { + "concept": { "type": "Relationship", "object": str() }, - "@context": dict() + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } self.keys = {k: k for k in self.data.keys()} @@ -96,13 +97,13 @@ def add_data(self, property_id, data): # Pending to implement in the Context Broker ############################################################################### # for i in range(0, len(languages)): - # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] + # self.data['label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['rdfs:label']['value'][languages[i]] = descriptions[i] + self.data['label']['value'][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['dct:language'] + key = self.keys['language'] self.data[key]['value'] = languages # qb:codeList, this attribute might not be presented, so we need to check it. @@ -110,18 +111,18 @@ def add_data(self, property_id, data): try: position = data.index('qb:codeList') + 1 code_list = self.generate_id(entity="ConceptSchema", value=data[position][0]) - self.data['qb:codeList']['object'] = code_list + self.data['codeList']['object'] = code_list except ValueError: logger.warning(f'Property: {property_id} has not qb:codeList, deleting the key in the data') # If we have not the property, we delete it from data - self.data.pop('qb:codeList') + self.data.pop('codeList') # qb:concept # TODO: the concept id need to check if it is a normal id or an url position = data.index('qb:concept') + 1 concept = self.generate_id(entity="Concept", value=data[position][0]) - self.data['qb:concept']['object'] = concept + self.data['concept']['object'] = concept # Get the rest of the data data = get_rest_data(data=data, @@ -146,7 +147,7 @@ def add_data(self, property_id, data): # Simplify Context and order keys a = Context() a.set_data(data=self.data) - a.new_analysis() + # a.new_analysis() a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index cc3481a..829dead 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -47,7 +47,7 @@ def prefixid(self, s): self.context.add_context(context) def triples(self, triple): - self.entity_type.set_context(context=self.get_context(), mapping=self.get_context_mapping()) + # self.entity_type.set_context(context=self.get_context(), mapping=self.get_context_mapping()) self.entity_type.transform(string=triple) return triple diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index d44a7f3..b8954ba 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -1,5 +1,6 @@ from unittest import TestCase -from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value +from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value, extract_prefix +from sdmx2jsonld.exceptions.exceptions import ClassExtractPrefixError class TestRegToParser(TestCase): def setUp(self) -> None: @@ -19,3 +20,39 @@ def test_flatten_value(self): got_data = flatten_value(data) assert(got_data == expected_res) + # data = ['', None, 'dct:created', 'dct:identifier', 'dct:modified', 'rdfs:range', 'uno', 'a:b:c'] + def test_extract_prefix_with_a_prefix(self): + data = 'a:b' + expected_res = 'b' + got_res = extract_prefix(data) + assert(got_res == expected_res) + + def test_extract_prefix_with_several_prefixes(self): + data = 'a:b:c' + expected = "Unexpected number of prefixes: 'a:b:c'" + with self.assertRaises(ClassExtractPrefixError) as error: + _ = extract_prefix(attribute=data) + + self.assertEqual(str(error.exception.message), expected) + + def test_extract_prefix_with_None_value(self): + data = None + expected = "Unexpected data received: 'None'" + with self.assertRaises(ClassExtractPrefixError) as error: + _ = extract_prefix(attribute=data) + + self.assertEqual(str(error.exception.message), expected) + + def test_extract_prefix_with_empty_value(self): + data = '' + expected = "Unexpected data received: ''" + with self.assertRaises(ClassExtractPrefixError) as error: + _ = extract_prefix(attribute=data) + + self.assertEqual(str(error.exception.message), expected) + + def test_extract_prefix_with_a_value_without_prefix(self): + data = 'a' + expected_res = 'a' + got_res = extract_prefix(data) + assert(got_res == expected_res) From 6a2eb47ab273ab1bd44a76432dabe68c289024cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 15:01:15 +0200 Subject: [PATCH 47/74] Resolution of prefix and context in the Concept class --- sdmx2jsonld/transform/concept.py | 32 ++++++++++++++-------------- sdmx2jsonld/transform/property.py | 1 - sdmx2jsonld/transform/transformer.py | 1 - 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index 7095522..778726b 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -35,11 +35,11 @@ def __init__(self): self.data = { "id": str(), "type": "Concept", - "dct:language": { + "language": { "type": "Property", "value": list() }, - "skos:inScheme": { + "inScheme": { "type": "Relationship", "object": str() }, @@ -57,13 +57,14 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "skos:prefLabel": { + "prefLabel": { "type": "Property", "value": dict() }, - - "@context": dict() + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } self.concept_id = str() @@ -111,10 +112,10 @@ def add_data(self, concept_id, data): # self.data['skos:prefLabel']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['skos:prefLabel']['value'][languages[i]] = descriptions[i] + self.data['prefLabel']['value'][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['dct:language'] + key = self.keys['language'] self.data[key]['value'] = languages # Add the id @@ -130,11 +131,10 @@ def add_data(self, concept_id, data): self.need_add_notation(data=data) # Simplify Context and order keys - #a = Context() - #a.set_data(data=self.data) - #a.new_analysis() - #a.order_context() - #self.data = a.get_data() + a = Context() + a.set_data(data=self.data) + a.order_context() + self.data = a.get_data() def get(self): return self.data @@ -171,16 +171,16 @@ def need_add_in_scheme(self, data): parser = RegParser() concept_schema = data[position][0] concept_schema = "urn:ngsi-ld:ConceptSchema:" + parser.obtain_id(concept_schema) - if self.data['skos:inScheme']['type'] == 'Relationship': - self.data['skos:inScheme']['object'] = concept_schema + if self.data['inScheme']['type'] == 'Relationship': + self.data['inScheme']['object'] = concept_schema else: - self.data['skos:inScheme']['value'] = concept_schema + self.data['inScheme']['value'] = concept_schema def need_add_notation(self, data): try: position = data.index('skos:notation') + 1 - self.data['skos:notation'] = { + self.data['notation'] = { 'type': 'Property', 'value': data[position][0][0].replace("\"", "") } diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index 4d00767..70401c6 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -147,7 +147,6 @@ def add_data(self, property_id, data): # Simplify Context and order keys a = Context() a.set_data(data=self.data) - # a.new_analysis() a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index 829dead..478c607 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -47,7 +47,6 @@ def prefixid(self, s): self.context.add_context(context) def triples(self, triple): - # self.entity_type.set_context(context=self.get_context(), mapping=self.get_context_mapping()) self.entity_type.transform(string=triple) return triple From 2c669f1c2cb0ea867b44985455767d8a3773ca61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 15:16:41 +0200 Subject: [PATCH 48/74] Resolve prefixes in ConceptSchema, SDMX-Concepts, SDMX-Attributes, and SDMX-Dimensions --- sdmx2jsonld/sdmxattributes/sdmxattribute.py | 21 ++++++-------- sdmx2jsonld/sdmxconcepts/cogconceptschema.py | 14 ++++----- sdmx2jsonld/sdmxconcepts/sdmxconcept.py | 18 +++++------- sdmx2jsonld/sdmxdimensions/frequency.py | 21 ++++++-------- sdmx2jsonld/sdmxdimensions/refarea.py | 21 ++++++-------- sdmx2jsonld/sdmxdimensions/timeperiod.py | 21 ++++++-------- sdmx2jsonld/transform/conceptschema.py | 30 ++++++++++---------- 7 files changed, 59 insertions(+), 87 deletions(-) diff --git a/sdmx2jsonld/sdmxattributes/sdmxattribute.py b/sdmx2jsonld/sdmxattributes/sdmxattribute.py index c2524f8..f9161f6 100644 --- a/sdmx2jsonld/sdmxattributes/sdmxattribute.py +++ b/sdmx2jsonld/sdmxattributes/sdmxattribute.py @@ -30,17 +30,17 @@ def __init__(self, entity_id, label, description, concept_id, identifier, entity self.data = { "id": f"urn:ngsi-ld:AttributeProperty:{entity_id}", "type": "AttributeProperty", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "rdfs:label": { + "label": { "type": "Property", "value": { "en": label, } }, - "dct:description": { + "description": { "type": "Property", "value": { "en": description, @@ -50,20 +50,15 @@ def __init__(self, entity_id, label, description, concept_id, identifier, entity "type": "Relationship", "object": f"urn:ngsi-ld:Concept:{concept_id}" }, - "dct:identifier": { + "identifier": { "type": "Property", "value": identifier }, - "rdfs:range": { + "range": { "type": "Property", "value": entity_range }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } diff --git a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py index e2cc765..a7f2dc1 100644 --- a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py +++ b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py @@ -32,21 +32,17 @@ def __init__(self): self.data = { "id": "urn:ngsi-ld:ConceptSchema:cog", "type": "ConceptScheme", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "skos:prefLabel": { + "prefLabel": { "type": "Property", "value": { "en": "Content Oriented Guidelines concept scheme" } }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } diff --git a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py index 25c1fae..4e19ade 100644 --- a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py +++ b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py @@ -28,29 +28,25 @@ def __init__(self, entity_id, label, notation): self.data = { "id": f"urn:ngsi-ld:Concept:{entity_id}", "type": "Concept", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "skos:inScheme": { + "inScheme": { "type": "Relationship", "object": "urn:ngsi-ld:ConceptSchema:cog" }, - "skos:prefLabel": { + "prefLabel": { "type": "Property", "value": { "en": label } }, - "skos:notation": { + "notation": { "type": "Property", "value": notation }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "skos": "http://www.w3.org/2004/02/skos/core#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py index 5f46a54..224ed12 100644 --- a/sdmx2jsonld/sdmxdimensions/frequency.py +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -36,17 +36,17 @@ def __init__(self): self.data = { "id": "urn:ngsi-ld:DimensionProperty:freq", "type": "DimensionProperty", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "rdfs:label": { + "label": { "type": "Property", "value": { "en": "Frequency", } }, - "dct:description": { + "description": { "type": "Property", "value": { "en": "The time interval at which observations occur over a given time period.", @@ -56,22 +56,17 @@ def __init__(self): "type": "Relationship", "object": "urn:ngsi-ld:Concept:freq" }, - "dct:identifier": { + "identifier": { "type": "Property", "value": "freq" }, - "rdfs:range": { + "range": { "type": "Property", "value": "xsd:string" }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } @staticmethod diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index eca97f7..f2a921b 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -34,17 +34,17 @@ def __init__(self): self.data = { "id": "urn:ngsi-ld:DimensionProperty:refArea", "type": "DimensionProperty", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "rdfs:label": { + "label": { "type": "Property", "value": { "en": "Reference Area", } }, - "dct:description": { + "description": { "type": "Property", "value": { "en": "The country or geographic area to which the measured statistical phenomenon relates.", @@ -54,20 +54,15 @@ def __init__(self): "type": "Relationship", "object": "urn:ngsi-ld:Concept:refArea" }, - "dct:identifier": { + "identifier": { "type": "Property", "value": "refArea" }, - "rdfs:range": { + "range": { "type": "Property", "value": "xsd:string" }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py index 303eb29..a17fc37 100644 --- a/sdmx2jsonld/sdmxdimensions/timeperiod.py +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -36,17 +36,17 @@ def __init__(self): self.data = { "id": "urn:ngsi-ld:DimensionProperty:timePeriod", "type": "DimensionProperty", - "dct:language": { + "language": { "type": "Property", "value": ["en"] }, - "rdfs:label": { + "label": { "type": "Property", "value": { "en": "Time Period", } }, - "dct:description": { + "description": { "type": "Property", "value": { "en": "The period of time or point in time to which the measured observation refers.", @@ -56,22 +56,17 @@ def __init__(self): "type": "Relationship", "object": "urn:ngsi-ld:Concept:timePeriod" }, - "dct:identifier": { + "identifier": { "type": "Property", "value": "timePeriod" }, - "rdfs:range": { + "range": { "type": "Property", "value": "xsd:string" }, - "@context": { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", - "dcat": "http://www.w3.org/ns/dcat#", - "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", - "dct": "http://purl.org/dc/terms/", - "qb": "http://purl.org/linked-data/cube#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#" - } + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } @staticmethod diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 8d8865a..656f464 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -35,11 +35,11 @@ def __init__(self): self.data = { "id": str(), "type": "ConceptScheme", - "dct:language": { + "language": { "type": "Property", "value": list() }, - "skos:hasTopConcept": { + "hasTopConcept": { "type": "Relationship", "object": list() }, @@ -54,18 +54,19 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "skos:prefLabel": { + "prefLabel": { "type": "Property", "value": dict() }, - "@context": dict() + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } self.keys = {k: k for k in self.data.keys()} - def add_data(self, concept_schema_id, data): # TODO: We have to control that data include the indexes that we want to search # We need to complete the data corresponding to the ConceptSchema: skos:prefLabel @@ -98,10 +99,10 @@ def add_data(self, concept_schema_id, data): # self.data['skos:prefLabel']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['skos:prefLabel']['value'][languages[i]] = descriptions[i] + self.data['prefLabel']['value'][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['dct:language'] + key = self.keys['language'] self.data[key]['value'] = languages # Add the id @@ -111,27 +112,26 @@ def add_data(self, concept_schema_id, data): # skos:hasTopConcept, this is a list of ids position = data.index('skos:hasTopConcept') + 1 result = list(map(lambda x: self.generate_id(value=x, entity='Concept'), data[position])) - self.data['skos:hasTopConcept']['object'] = result + self.data['hasTopConcept']['object'] = result # Get the rest of data, dct:created and dct:modified properties position = data.index('dct:created') + 1 - self.data['dct:created'] = { + self.data['created'] = { "type": "Property", "value": flatten_value(data[position]) } position = data.index('dct:modified') + 1 - self.data['dct:modified'] = { + self.data['modified'] = { "type": "Property", "value": flatten_value(data[position]) } # Simplify Context and order keys - # a = Context() - # a.set_data(data=self.data) - # a.new_analysis() - # a.order_context() - # self.data = a.get_data() + a = Context() + a.set_data(data=self.data) + a.order_context() + self.data = a.get_data() def get(self): return self.data From ad0dcfe7bccddec95c6f439ee8d1e6950999a74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 20 Jun 2023 18:49:59 +0200 Subject: [PATCH 49/74] Resolve the Catalogue prefix management and update class following the update of statDCAT-AP:Catalogue data model --- sdmx2jsonld/common/listmanagement.py | 19 +++++++ sdmx2jsonld/transform/catalogue.py | 34 ++++++----- sdmx2jsonld/transform/distribution.py | 4 +- tests/common/test_listmanagement.py | 82 ++++++++++++++++++++++++++- 4 files changed, 121 insertions(+), 18 deletions(-) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index b260400..e80ee7e 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -107,3 +107,22 @@ def extract_prefix(attribute): raise ClassExtractPrefixError(data=attribute, message=f"Unexpected number of prefixes: '{attribute}'") return result + + +def get_property_value(data, property_name): + # At the moment, we only find the first occurs of the property + i = 0 + key = '' + found = -1 + for i in range(0, len(data)): + key = data[i] + if isinstance(key, str): + found = key.find(property_name) + if found != -1: + # We found the key + break + + if found != -1: + return i, key, data[i+1] + else: + return -1, '', '' diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index bbf5a72..3c321c4 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -22,7 +22,7 @@ from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass -from sdmx2jsonld.common.listmanagement import get_rest_data +from sdmx2jsonld.common.listmanagement import get_rest_data, get_property_value from sdmx2jsonld.transform.context import Context from random import getrandbits @@ -37,12 +37,12 @@ def __init__(self): "id": str(), "type": "Catalogue", - "qb:dataset": { + "dataset": { "type": "Relationship", "object": str() }, - "dct:language": { + "language": { "type": "Property", "value": list() }, @@ -57,18 +57,18 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "dct:description": { + "description": { "type": "Property", "value": dict() }, - "dct:publisher": { + "publisher": { "type": "Property", "value": str() }, - "dct:title": { + "title": { "type": "Property", "value": list() }, @@ -93,16 +93,14 @@ def add_dataset(self, dataset_id): self.data['id'] = "urn:ngsi-ld:Catalogue:" + hash1 # Add dataset id - - self.data['qb:dataset']['object'] = dataset_id - + self.data['dataset']['object'] = dataset_id def add_data(self, title, dataset_id, data): # We need to complete the data corresponding to the Catalogue: rdfs:label self.__complete_label__(title=title, data=data) # Add the title - key = self.keys['dct:title'] + key = self.keys['title'] self.data[key]['value'] = title # Add the id @@ -111,11 +109,17 @@ def add_data(self, title, dataset_id, data): # Add the publisher key = self.get_key(requested_key='dcterms:publisher') position = data.index(key) + 1 - self.data['dct:publisher']['value'] = data[position][0] + self.data['publisher']['value'] = data[position][0] + + # Check if we have 'issued' in the original, then we need to create the releaseDate property + index, key, value = get_property_value(data=data, property_name='issued') + if index != -1: + # We found an 'issued' data + self.data.update(self.__generate_property__(key='releaseDate', value=value[0][0])) - # Get the rest of the data, qb:structure has the same value of qb:dataset so we decide to + # Get the rest of the data, qb:structure has the same value of qb:dataset, so we decide to # use only qb:dataset in CatalogueDCAT-AP - data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher', 'structure']) + data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher', 'structure', 'issued', 'title']) # add the new data to the dataset structure self.patch_data(data, False) @@ -165,11 +169,11 @@ def __complete_label__(self, title, data): # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - key = self.keys['dct:description'] + key = self.keys['description'] self.data[key]['value'][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['dct:language'] + key = self.keys['language'] self.data[key]['value'] = languages except ValueError: logger.info(f'Dataset without rdfs:label detail: {title}') diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index d9916a0..62588aa 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -81,10 +81,10 @@ def generate_data(self, catalogue): self.data['id'] += hash1 # Title is extracted from the dcterms:title from the Catalogue - self.data['Title'] = catalogue.data['dcterms:title']['value'] + self.data['Title'] = catalogue.data['title']['value'] # language es obtained from language from the Catalogue - self.data['language'] = catalogue.data['dct:language']['value'] + self.data['language'] = catalogue.data['language']['value'] # accessURL is generated from the configuration file. config_path = Path.cwd().joinpath('common/config.json') diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index b8954ba..8c8639b 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -1,5 +1,5 @@ from unittest import TestCase -from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value, extract_prefix +from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value, extract_prefix, get_property_value from sdmx2jsonld.exceptions.exceptions import ClassExtractPrefixError class TestRegToParser(TestCase): @@ -56,3 +56,83 @@ def test_extract_prefix_with_a_value_without_prefix(self): expected_res = 'a' got_res = extract_prefix(data) assert(got_res == expected_res) + + def test_get_property_data_from_array_property_without_prefix(self): + data = ['a', + ['qb:DataSet'], + 'rdfs:label', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'dcterms:issued', + [['2022-04-01T06:00:00+00:00']], + 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], + 'dcterms:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'qb:structure', + ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], + 'sdmx-attribute:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']] + ] + expected = [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']] + + index, key, obtained = get_property_value(data=data, property_name='title') + + self.assertEqual(index, 8) + self.assertEqual(key, 'dcterms:title') + self.assertEqual(expected, obtained) + + def test_get_property_data_from_array_property_with_prefix(self): + data = ['a', + ['qb:DataSet'], + 'rdfs:label', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'dcterms:issued', + [['2022-04-01T06:00:00+00:00']], + 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], + 'dcterms:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'qb:structure', + ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], + 'sdmx-attribute:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']] + ] + expected = [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']] + + index, key, obtained = get_property_value(data=data, property_name='dcterms:title') + + self.assertEqual(index, 8) + self.assertEqual(key, 'dcterms:title') + self.assertEqual(expected, obtained) + + def test_get_property_data_from_array_invalid_property(self): + data = ['a', + ['qb:DataSet'], + 'rdfs:label', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'dcterms:issued', + [['2022-04-01T06:00:00+00:00']], + 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], + 'dcterms:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']], + 'qb:structure', + ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], + 'sdmx-attribute:title', + [['"GDP and main components (current prices)"', '@en'], + ['"PIB et principales composantes (prix courants)"', '@fr']] + ] + expected = '' + + index, key, obtained = get_property_value(data=data, property_name='any') + + self.assertEqual(index, -1) + self.assertEqual(key, '') + self.assertEqual(expected, obtained) From cd39ee17e188020191206b2d550d36d622505989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 21 Jun 2023 08:46:57 +0200 Subject: [PATCH 50/74] Resolution of the Distribution class (prefix) and fixing data model representation --- sdmx2jsonld/transform/distribution.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index 62588aa..7d5f71f 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -32,17 +32,14 @@ class Distribution(CommonClass): def __init__(self): super().__init__(entity="Distribution") self.data = { - "id": "urn:ngsi-ld:DistributionDCAT-AP:", + "id": "urn:ngsi-ld:Distribution:", + "type": "Distribution", "accessUrl": { "type": "Property", "value": [ "/ngsi-ld/v1/entities?type=https://smartdatamodels.org/dataModel.SDMX/Observation" ] }, - "availability": { - "type": "Property", - "value": "STABLE" - }, "description": { "type": "Property", "value": "Distribution of statistical data observations." @@ -51,12 +48,6 @@ def __init__(self): "type": "Property", "value": "JSON_LD" }, - "accessService": { - "type": "Property", - "value": [ - "Orion-LD" - ] - }, "language": { "type": "Property", "value": list() @@ -65,12 +56,12 @@ def __init__(self): "type": "Property", "value": "Completed" }, - "Title": { + "title": { "type": "Property", "value": list() }, "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.DCAT-AP/master/context.jsonld" + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" ] } @@ -81,7 +72,7 @@ def generate_data(self, catalogue): self.data['id'] += hash1 # Title is extracted from the dcterms:title from the Catalogue - self.data['Title'] = catalogue.data['title']['value'] + self.data['title']['value'] = catalogue.data['title']['value'] # language es obtained from language from the Catalogue self.data['language'] = catalogue.data['language']['value'] From 7256fce43b0ab6d1ddaa5c43469a4683b1f62a98 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 10:06:41 +0200 Subject: [PATCH 51/74] changed dockerfile --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 582af3e..0d5074a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,4 +41,3 @@ WORKDIR $INSTALLATION_PATH EXPOSE ${PORT} ENTRYPOINT /usr/local/bin/python agent.py server --host 0.0.0.0 --port ${PORT} -expose $PORT From 2dc150ff0a4ffaa1ea164e1130caf85d3e139ff7 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 10:10:34 +0200 Subject: [PATCH 52/74] changed version for other things --- docker/Dockerfile | 21 ++++++++++++++++++--- docker/Readme.md | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0d5074a..d04b844 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,11 +2,11 @@ FROM alpine/git as git-clone ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle -ARG VERSION=develop +ARG BRANCH=develop ARG INSTALLATION_PATH=/opt/$COMPONENT RUN mkdir -p $INSTALLATION_PATH -RUN git clone https://github.com/$PROJECT/$COMPONENT --branch $VERSION $INSTALLATION_PATH +RUN git clone https://github.com/$PROJECT/$COMPONENT --branch $BRANCH $INSTALLATION_PATH ## Install PIP Requirements @@ -14,7 +14,6 @@ FROM python:3.11-alpine as pip-requirements ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle -ARG VERSION=integration-cb-httpserver ARG INSTALLATION_PATH=/opt/$COMPONENT RUN mkdir -p $INSTALLATION_PATH @@ -24,6 +23,22 @@ RUN pip install --root-user-action=ignore --prefix=$INSTALLATION_PATH -r /requir FROM python:3.11-alpine as final +LABEL "maintainer"="FIWARE Foundation e.V." +LABEL "description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." +LABEL "name"="iotagent-turtle" +LABEL "summary"="IoT Agent for the DataQube representation in RDF Turtle format used in statistical environments" + +LABEL "org.opencontainers.image.authors"="fernando.lopez@fiware.org" +LABEL "org.opencontainers.image.documentation"="" +LABEL "org.opencontainers.image.vendor"="FIWARE Foundation e.V." +LABEL "org.opencontainers.image.licenses"="Apache2.0" +LABEL "org.opencontainers.image.title"="iotagent-turtle" +LABEL "org.opencontainers.image.description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." +LABEL "org.opencontainers.image.source"="https://github.com/flopezag/IoTAgent-Turtle" +LABEL "org.opencontainers.image.version"="0.1.0" +LABEL "org.python.version"="python:3.9" + + ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle ARG INSTALLATION_PATH=/opt/$COMPONENT diff --git a/docker/Readme.md b/docker/Readme.md index 158c330..7abd32f 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -9,7 +9,7 @@ The docker is easyly build using the following command - By default it will buil If we want to build other branch, we could define the branch we want to use this way: ``` - docker build . -t iotagent-turtle --build-arg VERSION=integration-cb-httpserver + docker build . -t iotagent-turtle --build-arg BRANCH=integration-cb-httpserver ``` Everything will install in /proc/IotAgent-turtle inside the container. The default config file will be the following one: From 697e105f94727217901e1afe781c7c018debcaa1 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 10:15:44 +0200 Subject: [PATCH 53/74] change documentation --- docker/Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Readme.md b/docker/Readme.md index 7abd32f..a55e0b9 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -28,12 +28,12 @@ Everything will install in /proc/IotAgent-turtle inside the container. The defau ``` ## Use -Let's suppose the image is named **io** and we want to expose port 5000 in order to work. We can also image that we want to remove the container when it finishes, we can do this way: +Let's suppose the image is named **iotagent-turtle** and we want to expose port 5000 in order to work. We can also image that we want to remove the container when it finishes, we can do it this way: ``` docker run --rm -p 5000:5000 --name io iotagent-turtle ``` -However, we might have Orion-ld somewhere else. So if we cant to connect our context broker somewhere, we can use this command: +However, we might have Orion-ld somewhere else. Therefore, we can provide the IP address of the host "orion-ld" where our Context Broker is listening this way: ``` docker run --rm -p 5000:5000 --name io --add-host=orion-ld:192.168.1.206 iotagent-turtle ``` From 9b5c1e421500630b7dc50830ddeb3c35e75f4cdd Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 10:28:49 +0200 Subject: [PATCH 54/74] finished all changes in the review --- docker/Readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Readme.md b/docker/Readme.md index a55e0b9..72287c1 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -1,7 +1,7 @@ # Docker for server ## build -The docker is easyly build using the following command - By default it will build the docker for the **develop** branch: +The docker is easyly build using the following command - By default it builds the docker for the **develop** branch: ``` docker build . -t iotagent-turtle @@ -39,14 +39,14 @@ docker run --rm -p 5000:5000 --name io --add-host=orion-ld:192.168.1.206 iotage ``` ### Overriding config.json -We could create our own config.json and override the one in the docker by default +We could create our own config.json and the default configuration file in the docker: ``` docker run --rm -p 5000:5000 --name io -v our-local-config.json:/opt/IoTAgent-turtle/common/config.json iotagent-turtle ``` ### As docker file -We can consider writing a docker-compose.yaml file as the following one to start everything (orion-ld, the name of our orion server is named according to our config.json file: +We can consider writing a docker-compose.yaml file as the following one to start everything (orion-ld, the name of our orion server is named according to our config.json file (see "broker" key): ``` version: "3.8" From 093dc93ad63752e8fb7dcd96e12f223bf8a80264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 21 Jun 2023 10:53:15 +0200 Subject: [PATCH 55/74] Resolution Dataset prefix and quality code checking --- api/server.py | 18 +++--- ngsild/ngsild_connector.py | 23 ++++---- sdmx2jsonld/common/classprecedence.py | 4 +- sdmx2jsonld/common/datatypeconversion.py | 1 + sdmx2jsonld/common/rdf.py | 4 +- sdmx2jsonld/sdmxattributes/code.py | 6 +- .../sdmxattributes/confirmationStatus.py | 5 +- .../sdmxattributes/observationStatus.py | 2 +- sdmx2jsonld/sdmxattributes/sdmxattribute.py | 2 - sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/decimals.py | 6 +- sdmx2jsonld/sdmxconcepts/dissorgconcept.py | 7 ++- sdmx2jsonld/sdmxconcepts/freqconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 6 +- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/sdmxconcept.py | 54 ++++++++--------- .../sdmxconcepts/timePerCollectConcept.py | 3 +- sdmx2jsonld/sdmxconcepts/timeformatconcept.py | 6 +- sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 6 +- sdmx2jsonld/sdmxconcepts/titleConcept.py | 3 +- sdmx2jsonld/sdmxconcepts/unitmultconcept.py | 3 +- sdmx2jsonld/transform/catalogue.py | 2 +- sdmx2jsonld/transform/concept.py | 4 +- sdmx2jsonld/transform/conceptschema.py | 4 +- sdmx2jsonld/transform/context.py | 7 ++- sdmx2jsonld/transform/dataset.py | 59 +++++++++---------- sdmx2jsonld/transform/entitytype.py | 17 +----- sdmx2jsonld/transform/observation.py | 1 - sdmx2jsonld/transform/property.py | 4 +- sdmx2jsonld/transform/transformer.py | 8 +-- 30 files changed, 137 insertions(+), 137 deletions(-) diff --git a/api/server.py b/api/server.py index 62d468c..4a8d90d 100644 --- a/api/server.py +++ b/api/server.py @@ -43,8 +43,8 @@ def create_app() -> FastAPI: app = FastAPI(title='IoTAgent-Turtle', debug=False) logging_config_path = Path.cwd().joinpath('common/config.json') - logger = CustomizeLogger.make_logger(logging_config_path) - app.logger = logger + customize_logger = CustomizeLogger.make_logger(logging_config_path) + app.logger = customize_logger return app @@ -58,12 +58,11 @@ async def set_secure_headers(request, call_next): server = Server().set("Secure") csp = ( - ContentSecurityPolicy() - .default_src("'none'") - .base_uri("'self'") - .connect_src("'self'" "api.spam.com") - .frame_src("'none'") - .img_src("'self'", "static.spam.com") + ContentSecurityPolicy().default_src("'none'") + .base_uri("'self'") + .connect_src("'self'" "api.spam.com") + .frame_src("'none'") + .img_src("'self'", "static.spam.com") ) hsts = StrictTransportSecurity().include_subdomains().preload().max_age(2592000) @@ -152,7 +151,6 @@ async def parse(request: Request, file: UploadFile, response: Response): } url = get_url() - resp = "..." try: request.app.logger.debug(f'Sending data:\n{json_object}') @@ -200,7 +198,7 @@ def get_uptime(): def get_url(): config_path = Path.cwd().joinpath('common/config.json') - config = dict() + with open(config_path) as config_file: config = load(config_file) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index 322f30d..d81b54a 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -21,17 +21,21 @@ ## from pathlib import Path import json -from requests import post, exceptions +from requests import post +from sdmx2jsonld.transform.parser import Parser +from io import StringIO + class NGSILDConnector: def __init__(self, path=None): - if path == None: + if path is None: config_path = Path.cwd().joinpath('common/config.json') else: config_path = Path.cwd().joinpath(path) - config = dict() + with open(config_path) as config_file: config = json.load(config_file) + self.base_url = config['broker'] def get_url(self): @@ -43,33 +47,30 @@ def send_data_array(self, json_object): d = d if type(d) is list else [d] for elem in d: - rc, r = c.send_data(json.dumps(elem)) + _, _ = c.send_data(json.dumps(elem)) def send_data(self, json_object): # Send the data to a FIWARE Context Broker instance headers = { 'Content-Type': 'application/ld+json' - #, 'Accept': 'application/ld+json' + # , 'Accept': 'application/ld+json' } url = self.get_url() resp = "..." - r = post(url=url, headers=headers, data=json_object, timeout=5) + response = post(url=url, headers=headers, data=json_object, timeout=5) # resp = json.loads(r.text) - response_status_code = r.status_code + response_status_code = response.status_code if response_status_code == 201: - print("LOCATION: ", r.headers['Location']) + print("LOCATION: ", response.headers['Location']) # Let exceptions raise.... They can be controlled somewhere else. return response_status_code, resp -from sdmx2jsonld.transform.parser import Parser -from io import StringIO - if __name__ == "__main__": c = NGSILDConnector('../common/config.json') print(c.get_url()) diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py index 95ca2f9..a912780 100644 --- a/sdmx2jsonld/common/classprecedence.py +++ b/sdmx2jsonld/common/classprecedence.py @@ -60,13 +60,13 @@ def precedence(self, data: list) -> str: classes_values = list(map(lambda x: self.get_value(x), data)) # We need to check if all element of the list are the value 250 because could not be possible to have at - # the same time a DimensionProperty and AttributeProperty, this is an ERROR and we need to report. + # the same time a DimensionProperty and AttributeProperty, this is an ERROR than we need to report. result = all(element == 250 for element in classes_values) and len(data) > 1 if result is True: raise ClassPrecedencePropertyError(data) # In case that we have several values identical of type Class, we need to report a WARNING message because maybe - # it is not needed multitype in that case. + # it is not needed multi-type in that case. result = all(element == 20 for element in classes_values) and len(data) > 1 if result is True: raise ClassPrecedenceClassError(data) diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index a4f3f3f..b444f6b 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -26,6 +26,7 @@ import pytz from sdmx2jsonld.common.tzinfos import whois_timezone_info + class DataTypeConversion: def __init__(self): self.types = { diff --git a/sdmx2jsonld/common/rdf.py b/sdmx2jsonld/common/rdf.py index 1eff170..65daa3c 100644 --- a/sdmx2jsonld/common/rdf.py +++ b/sdmx2jsonld/common/rdf.py @@ -23,10 +23,10 @@ def turtle_terse(rdf_content): - ''' + """ Function which receives a string in formatted with RDF, dialect turtle, parses it and returns the same data parsed in another string. The incoming data in the parameter is equivalent to the returned data. - ''' + """ # Create a Graph g2 = Graph() diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index dbe9374..506d2d9 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -24,9 +24,9 @@ class Code: - status: list() - type: str() - data_range: list() + status: list + type: str + data_range: range def __init__(self, typecode): self.typecode = typecode diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index 6defb97..dcf583f 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -25,7 +25,7 @@ class ConfStatus(SDMXAttribute): - status: list() = [ + status: list = [ "F", "N", "C", @@ -49,7 +49,8 @@ def __init__(self): # rdfs:isDefinedBy . super().__init__(entity_id='confStatus', label='Confidentiality - status', - description='Information about the confidentiality status of the object to which this attribute is attached.', + description='Information about the confidentiality status of the object ' + 'to which this attribute is attached.', concept_id='confStatus', identifier='confStatus', entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index 6ff2559..2410a54 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -25,7 +25,7 @@ class ObsStatus(SDMXAttribute): - status: list() = [ + status: list = [ "A", "B", "D", diff --git a/sdmx2jsonld/sdmxattributes/sdmxattribute.py b/sdmx2jsonld/sdmxattributes/sdmxattribute.py index f9161f6..2f587fa 100644 --- a/sdmx2jsonld/sdmxattributes/sdmxattribute.py +++ b/sdmx2jsonld/sdmxattributes/sdmxattribute.py @@ -19,8 +19,6 @@ # License for the specific language governing permissions and limitations # under the License. ## -from re import search -from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError from sdmx2jsonld.common.commonclass import CommonClass diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py index d27d733..df7250e 100644 --- a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -35,4 +35,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='confStatus', label='Confidentiality - status', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS') diff --git a/sdmx2jsonld/sdmxconcepts/decimals.py b/sdmx2jsonld/sdmxconcepts/decimals.py index 13770f0..c357189 100644 --- a/sdmx2jsonld/sdmxconcepts/decimals.py +++ b/sdmx2jsonld/sdmxconcepts/decimals.py @@ -28,8 +28,10 @@ def __init__(self): # rdfs:label "Decimals"@en ; # rdfs:comment """The number of digits of an observation to the right of a decimal point."""@en ; # rdfs:isDefinedBy ; - # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS"; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: + # CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS"; # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='decimals', label='Decimals', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS') diff --git a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py index 9b21c63..a81659a 100644 --- a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py +++ b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py @@ -28,9 +28,10 @@ def __init__(self): # rdfs:label "Data Dissemination Agency"@en ; # rdfs:comment """The organisation disseminating the data."""@en ; # rdfs:isDefinedBy ; - # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG"; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: + # CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG"; # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='dissOrg', label='Data Dissemination Agency', - notation= - 'urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG') diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index 891e362..ba97a74 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -32,4 +32,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog. super().__init__(entity_id='freq', label='Frequency', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].FREQ') diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py index 5639632..f3acf69 100644 --- a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -28,8 +28,10 @@ def __init__(self): # rdfs:label "Observation Status"@en ; # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; # rdfs:isDefinedBy ; - # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: + # CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='obsStatus', label='Observation Status', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS') diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index ac77804..22afa08 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -32,4 +32,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='refArea', label='Reference Area', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA') diff --git a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py index 4e19ade..3163659 100644 --- a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py +++ b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py @@ -23,30 +23,30 @@ class SDMXConcept(CommonClass): - def __init__(self, entity_id, label, notation): - super().__init__(entity='Concept') - self.data = { - "id": f"urn:ngsi-ld:Concept:{entity_id}", - "type": "Concept", - "language": { - "type": "Property", - "value": ["en"] - }, - "inScheme": { - "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "prefLabel": { - "type": "Property", - "value": { - "en": label - } - }, - "notation": { - "type": "Property", - "value": notation - }, - "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] - } + def __init__(self, entity_id, label, notation): + super().__init__(entity='Concept') + self.data = { + "id": f"urn:ngsi-ld:Concept:{entity_id}", + "type": "Concept", + "language": { + "type": "Property", + "value": ["en"] + }, + "inScheme": { + "type": "Relationship", + "object": "urn:ngsi-ld:ConceptSchema:cog" + }, + "prefLabel": { + "type": "Property", + "value": { + "en": label + } + }, + "notation": { + "type": "Property", + "value": notation + }, + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] + } diff --git a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py index 5e66a4a..3a65d8f 100644 --- a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py +++ b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py @@ -36,4 +36,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='timePerCollect', label='Time Period - collection', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT') diff --git a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py index 73ed042..fc2ab45 100644 --- a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py @@ -28,8 +28,10 @@ def __init__(self): # rdfs:label "Time Format"@en ; # rdfs:comment """Technical format in which time is represented for the measured phenomenon."""@en ; # rdfs:isDefinedBy ; - # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT"; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: + # CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT"; # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='timeFormat', label='Time Format', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT') diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index 9c00df6..50c83ea 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -28,8 +28,10 @@ def __init__(self): # rdfs:label "Time Period"@en; # rdfs:comment """The period of time or point in time to which the measured observation refers."""@en; # rdfs:isDefinedBy ; - # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; + # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: + # CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; # skos:inScheme sdmx-concept:cog. super().__init__(entity_id='timePeriod', label='Time Period', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD') diff --git a/sdmx2jsonld/sdmxconcepts/titleConcept.py b/sdmx2jsonld/sdmxconcepts/titleConcept.py index cde7540..bf65f68 100644 --- a/sdmx2jsonld/sdmxconcepts/titleConcept.py +++ b/sdmx2jsonld/sdmxconcepts/titleConcept.py @@ -32,4 +32,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='title', label='Title', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TITLE') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].TITLE') diff --git a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py index b89c25d..de24d9b 100644 --- a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py +++ b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py @@ -34,4 +34,5 @@ def __init__(self): # skos:inScheme sdmx-concept:cog . super().__init__(entity_id='unitMult', label='Unit Multiplier', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT') + notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' + 'CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT') diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 3c321c4..7d77350 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -126,7 +126,7 @@ def add_data(self, title, dataset_id, data): # Order Context keys a = Context() - a.set_data(data=self.data) + a.set_data(new_data=self.data) a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index 778726b..b4dddd1 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -130,9 +130,9 @@ def add_data(self, concept_id, data): # skos:notation self.need_add_notation(data=data) - # Simplify Context and order keys + # Order the keys in the final json-ld a = Context() - a.set_data(data=self.data) + a.set_data(new_data=self.data) a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 656f464..4ff58d0 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -127,9 +127,9 @@ def add_data(self, concept_schema_id, data): "value": flatten_value(data[position]) } - # Simplify Context and order keys + # Order the keys in the final json-ld a = Context() - a.set_data(data=self.data) + a.set_data(new_data=self.data) a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/context.py b/sdmx2jsonld/transform/context.py index 6465af1..e0e0fe5 100644 --- a/sdmx2jsonld/transform/context.py +++ b/sdmx2jsonld/transform/context.py @@ -57,6 +57,7 @@ def add_context(self, context): value = aux[0][1] found = False + k = '' # check if the value of the new_context is in one of the values of the previous context for k, v in self.context['@context'].items(): @@ -150,8 +151,8 @@ def order_context(self): def get_data(self): return self.data - def set_data(self, data): - self.data = data + def set_data(self, new_data): + self.data = new_data if __name__ == '__main__': @@ -176,7 +177,7 @@ def set_data(self, data): 'dc:modified': '2022-01-15T10:00:00+00:00', 'other_thing': 'foo'} - a.set_data(data=data) + a.set_data(new_data=data) a.new_analysis() print(a.get_data()) a.order_context() diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index c3ba757..c2d9a7b 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -75,9 +75,12 @@ def __init__(self): self.data = { "id": str(), "type": "Dataset", - "dct:title": str(), - "dct:identifier": str(), - "dct:language": { + "title": { + "type": "Property", + "value": str() + }, + "identifier": str(), + "language": { "type": "Property", "value": list() }, @@ -92,21 +95,23 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "dct:description": { + "description": { "type": "Property", "value": dict() }, - "@context": dict() + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] } self.components = { 'qb:attribute': { 'entity': 'AttributeProperty', - 'key': 'stat:attribute', + 'key': 'attribute', 'value': { - "stat:attribute": { + "attribute": { "type": "Relationship", "object": list() } @@ -114,19 +119,19 @@ def __init__(self): }, 'qb:dimension': { 'entity': 'DimensionProperty', - 'key': 'stat:dimension', + 'key': 'dimension', 'value': { - "stat:dimension": { + "dimension": { "type": "Relationship", "object": list() } } }, 'qb:measure': { - 'entity': 'Measure', - 'key': 'stat:statUnitMeasure', + 'entity': 'statUnitMeasure', + 'key': 'statUnitMeasure', 'value': { - "stat:statUnitMeasure": { + "statUnitMeasure": { "type": "Relationship", "object": list() } @@ -183,7 +188,7 @@ def __init__(self): self.sdmx_concept_schemas = CogConceptSchema() - def add_components(self, context, component): + def add_components(self, component): # We need to know which kind of component we have, it should be the verb: # qb:attribute, qb:dimension, or qb:measure list_components = ['qb:attribute', 'qb:dimension', 'qb:measure'] @@ -197,12 +202,11 @@ def add_components(self, context, component): else: new_component, new_concept, new_concept_schema = self.manage_components(type_component=type_component, component=component, - position=position, - context=context) + position=position) return new_component, new_concept, new_concept_schema - def manage_components(self, type_component, component, position, context): + def manage_components(self, type_component, component, position): new_component, new_concept, new_concept_schema = None, None, None try: entity = self.components[type_component]['entity'] @@ -230,16 +234,11 @@ def manage_components(self, type_component, component, position, context): except ValueError: logger.error(f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}") - # Simplify Context and order keys. It is possible that we call add_component before the dataset has been created - # therefore we need to add the corresponding context to the dataset - # if len(self.data['@context']) == 0: - # self.data['@context'] = context['@context'] - # - # a = Context() - # a.set_data(data=self.data) - # a.new_analysis() - # a.order_context() - # self.data = a.get_data() + # Order the keys in the final json-ld + a = Context() + a.set_data(new_data=self.data) + a.order_context() + self.data = a.get_data() return new_component, new_concept, new_concept_schema @@ -251,8 +250,8 @@ def add_data(self, title, dataset_id, data): self.__complete_label__(title=title, data=data) # Add the title - key = self.keys['dct:title'] - self.data[key] = title + key = self.keys['title'] + self.data[key]['value'] = title # Add the id self.data['id'] = "urn:ngsi-ld:Dataset:" + dataset_id @@ -314,11 +313,11 @@ def __complete_label__(self, title, data): # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - key = self.keys['dct:description'] + key = self.keys['description'] self.data[key]['value'][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['dct:language'] + key = self.keys['language'] self.data[key]['value'] = languages except ValueError: logger.info(f'DataStructureDefinition without rdfs:label detail: {title}') diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index f7741e5..27fec00 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -69,8 +69,6 @@ def __find_entity_type__(self, string): """ Find the index position of the 'a' SDMX key and return the following data with the corresponding EntityType """ - is_new = bool() - # Index maybe 0 in case of ComponentSpecification or 1 in case of DataStructureDefinition index = len(string) - 1 string1 = string[index] @@ -80,6 +78,7 @@ def __find_entity_type__(self, string): # created. try: position = string1.index('a') + 1 + data = '' try: data = self.pre.precedence(string1[position]) @@ -110,10 +109,6 @@ def __find_entity_type__(self, string): return data, string1, is_new def transform(self, string): - #if len(self.context) == 0: - # raise AssertionError("Context should be passed before to the EntityType Class, " - # "call EntityType.set_context() before, {'__file__': this_file}))") - data_type, new_string, is_new = self.__find_entity_type__(string=string) if is_new: @@ -145,7 +140,7 @@ def flatten_value(y): def create_data(self, entity_type, data, title): if entity_type == 'Component': some_new_component, some_new_concept, some_new_concept_schema = \ - self.dataset.add_components(context=self.context, component=data) + self.dataset.add_components(component=data) if some_new_component is not None: if some_new_component.data['type'] == 'DimensionProperty': @@ -160,7 +155,7 @@ def create_data(self, entity_type, data, title): f'type: {some_new_component.data["type"]}') self.conceptLists.append(some_new_concept) - #self.conceptListsIds[title] = concept_list.get_id() + # we need to check that the conceptSchema is not already defined in the structure if some_new_concept_schema not in self.conceptSchemas: self.conceptSchemas.append(some_new_concept_schema) @@ -174,33 +169,28 @@ def create_data(self, entity_type, data, title): self.observations.append(observation) elif entity_type == 'Dataset': identifier = self.parser.obtain_id(title) - #self.dataset.add_context(context=self.context, context_mapping=self.context_mapping) self.dataset.add_data(title=title, dataset_id=identifier, data=data) # Create the CatalogueDCAT-AP and assign the dataset id self.catalogue.add_dataset(dataset_id=self.dataset.data['id']) elif entity_type == 'Dimension': dimension = Dimension() - #dimension.add_context(context=self.context, context_mapping=self.context_mapping) dimension_id = self.parser.obtain_id(title) dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) elif entity_type == 'Attribute': attribute = Attribute() - #attribute.add_context(context=self.context, context_mapping=self.context_mapping) attribute_id = self.parser.obtain_id(title) attribute.add_data(attribute_id=attribute_id, data=data) self.attributes.append(attribute) elif entity_type == 'ConceptScheme': concept_schema = ConceptSchema() - #concept_schema.add_context(context=self.context, context_mapping=self.context_mapping) concept_schema_id = self.parser.obtain_id(title) concept_schema.add_data(concept_schema_id=concept_schema_id, data=data) self.conceptSchemas.append(concept_schema) elif entity_type == 'Class': # We need the Concept because each of the Range description is of the type Concept concept_list = Concept() - #concept_list.add_context(context=self.context, context_mapping=self.context_mapping) concept_list_id = self.parser.obtain_id(title) concept_list.add_data(concept_id=concept_list_id, data=data) self.conceptLists.append(concept_list) @@ -208,7 +198,6 @@ def create_data(self, entity_type, data, title): elif entity_type == 'Range': # TODO: Range is associated to a Concept and identified properly in the ConceptSchema data_range = Concept() - #data_range.add_context(context=self.context, context_mapping=self.context_mapping) data_range_id = self.parser.obtain_id(title) data_range.add_data(concept_id=data_range_id, data=data) self.conceptLists.append(data_range) diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index a25345a..867a280 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -189,7 +189,6 @@ def get_key(self, requested_key): m = search('(.*):(.*)', requested_key) if m is not None: - prefix = m.group(1) subfix = m.group(2) try: diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index 70401c6..caf479b 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -144,9 +144,9 @@ def add_data(self, property_id, data): # add the new data to the dataset structure [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] - # Simplify Context and order keys + # Order the keys in the final json-ld a = Context() - a.set_data(data=self.data) + a.set_data(new_data=self.data) a.order_context() self.data = a.get_data() diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index 478c607..42e36f1 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -20,7 +20,7 @@ # under the License. ## -from lark import Transformer, Tree, Token +from lark import Transformer from sdmx2jsonld.transform.context import Context from sdmx2jsonld.transform.entitytype import EntityType from sdmx2jsonld.common.datatypeconversion import DataTypeConversion @@ -35,7 +35,6 @@ def __init__(self): self.entity_type = EntityType() # Regex to check valid URL - # regex = "" regex = "http[s]?:\/\/(.*)" # Compile the Regex @@ -93,8 +92,8 @@ def iri(self, iri): def verb(self, verb): return str(verb[0]) - def object(self, object): - return object[0] + def object(self, my_object): + return my_object[0] def literal(self, literal): return literal[0] @@ -162,4 +161,3 @@ def save(self): distribution.generate_data(catalogue=self.entity_type.catalogue) distribution.save() - From 63e157d42497e4943f7daca00b25be2b55beb775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 21 Jun 2023 12:35:27 +0200 Subject: [PATCH 56/74] Adding measureType as a new dimension defined externally to the parser file --- sdmx2jsonld/cube/__init__.py | 0 sdmx2jsonld/cube/measuretype.py | 41 ++++++++++++ sdmx2jsonld/sdmxdimensions/frequency.py | 46 +++---------- sdmx2jsonld/sdmxdimensions/refarea.py | 47 +++----------- sdmx2jsonld/sdmxdimensions/sdmxdimension.py | 72 +++++++++++++++++++++ sdmx2jsonld/sdmxdimensions/timeperiod.py | 46 +++---------- sdmx2jsonld/transform/dataset.py | 10 ++- sdmx2jsonld/transform/entitytype.py | 3 +- 8 files changed, 147 insertions(+), 118 deletions(-) create mode 100644 sdmx2jsonld/cube/__init__.py create mode 100644 sdmx2jsonld/cube/measuretype.py create mode 100644 sdmx2jsonld/sdmxdimensions/sdmxdimension.py diff --git a/sdmx2jsonld/cube/__init__.py b/sdmx2jsonld/cube/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sdmx2jsonld/cube/measuretype.py b/sdmx2jsonld/cube/measuretype.py new file mode 100644 index 0000000..610691a --- /dev/null +++ b/sdmx2jsonld/cube/measuretype.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from re import search +from sdmx2jsonld.exceptions.exceptions import ClassFreqError +from sdmx2jsonld.sdmxdimensions.sdmxdimension import SDMXDimension + + +class MeasureType(SDMXDimension): + def __init__(self): + # qb:measureType a qb:DimensionProperty, rdf:Property; + # rdfs:label "measure type"@en; + # rdfs:comment "Generic measure dimension, the value of this dimension indicates which measure (from the set of measures in the DSD) is being given by the obsValue (or other primary measure)"@en; + # rdfs:range qb:MeasureProperty; + # rdfs:isDefinedBy ; + super().__init__(entity_id='measureType', + label='measure type', + description='Generic measure dimension, the value of this dimension indicates which measure ' + '(from the set of measures in the DSD) is being given by the obsValue ' + '(or other primary measure).', + concept_id=None, + identifier='measureType', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py index 224ed12..6446d2f 100644 --- a/sdmx2jsonld/sdmxdimensions/frequency.py +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -21,10 +21,10 @@ ## from re import search from sdmx2jsonld.exceptions.exceptions import ClassFreqError -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxdimensions.sdmxdimension import SDMXDimension -class Frequency(CommonClass): +class Frequency(SDMXDimension): def __init__(self): # sdmx-dimension:freq a qb:DimensionProperty, rdf:Property; # rdfs:range rdfs:Resource; @@ -32,42 +32,12 @@ def __init__(self): # rdfs:label "Frequency"@en; # rdfs:comment """The time interval at which observations occur over a given time period."""@en; # rdfs:isDefinedBy . - super().__init__(entity='DimensionProperty') - self.data = { - "id": "urn:ngsi-ld:DimensionProperty:freq", - "type": "DimensionProperty", - "language": { - "type": "Property", - "value": ["en"] - }, - "label": { - "type": "Property", - "value": { - "en": "Frequency", - } - }, - "description": { - "type": "Property", - "value": { - "en": "The time interval at which observations occur over a given time period.", - } - }, - "concept": { - "type": "Relationship", - "object": "urn:ngsi-ld:Concept:freq" - }, - "identifier": { - "type": "Property", - "value": "freq" - }, - "range": { - "type": "Property", - "value": "xsd:string" - }, - "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] - } + super().__init__(entity_id='freq', + label='Frequency', + description='The time interval at which observations occur over a given time period.', + concept_id='freq', + identifier='freq', + entity_range='xsd:string') @staticmethod def fix_value(value): diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index f2a921b..8e70680 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -19,10 +19,10 @@ # License for the specific language governing permissions and limitations # under the License. ## -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxdimensions.sdmxdimension import SDMXDimension -class RefArea(CommonClass): +class RefArea(SDMXDimension): def __init__(self): # sdmx-dimension:refArea a qb:DimensionProperty, rdf:Property; # rdfs:range rdfs:Resource; @@ -30,39 +30,10 @@ def __init__(self): # rdfs:label "Reference Area"@en; # rdfs:comment "The country or geographic area to which the measured statistical phenomenon relates."@en; # rdfs:isDefinedBy . - super().__init__(entity='DimensionProperty') - self.data = { - "id": "urn:ngsi-ld:DimensionProperty:refArea", - "type": "DimensionProperty", - "language": { - "type": "Property", - "value": ["en"] - }, - "label": { - "type": "Property", - "value": { - "en": "Reference Area", - } - }, - "description": { - "type": "Property", - "value": { - "en": "The country or geographic area to which the measured statistical phenomenon relates.", - } - }, - "concept": { - "type": "Relationship", - "object": "urn:ngsi-ld:Concept:refArea" - }, - "identifier": { - "type": "Property", - "value": "refArea" - }, - "range": { - "type": "Property", - "value": "xsd:string" - }, - "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] - } + super().__init__(entity_id='refArea', + label='Reference Area', + description='The country or geographic area to which the measured statistical ' + 'phenomenon relates.', + concept_id='refArea', + identifier='refArea', + entity_range='xsd:string') diff --git a/sdmx2jsonld/sdmxdimensions/sdmxdimension.py b/sdmx2jsonld/sdmxdimensions/sdmxdimension.py new file mode 100644 index 0000000..8a59390 --- /dev/null +++ b/sdmx2jsonld/sdmxdimensions/sdmxdimension.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## +from sdmx2jsonld.common.commonclass import CommonClass + + +class SDMXDimension(CommonClass): + def __init__(self, entity_id, identifier, entity_range, label=None, description=None, concept_id=None): + super().__init__(entity='DimensionProperty') + self.data = { + "id": f"urn:ngsi-ld:DimensionProperty:{entity_id}", + "type": "DimensionProperty", + "language": { + "type": "Property", + "value": ["en"] + }, + "label": { + "type": "Property", + "value": { + "en": label, + } + }, + "description": { + "type": "Property", + "value": { + "en": description, + } + }, + "concept": { + "type": "Relationship", + "object": f"urn:ngsi-ld:Concept:{concept_id}" + }, + "identifier": { + "type": "Property", + "value": identifier + }, + "range": { + "type": "Property", + "value": entity_range + }, + "@context": [ + "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" + ] + } + + # We need to check if some of the parameters are None, in that case we have to pop the key from data + if label is None: + self.data.pop('label') + + if description is None: + self.data.pop('description') + + if concept_id is None: + self.data.pop('concept') diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py index a17fc37..24e4479 100644 --- a/sdmx2jsonld/sdmxdimensions/timeperiod.py +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -21,10 +21,10 @@ ## from re import search from sdmx2jsonld.exceptions.exceptions import ClassFreqError -from sdmx2jsonld.common.commonclass import CommonClass +from sdmx2jsonld.sdmxdimensions.sdmxdimension import SDMXDimension -class TimePeriod(CommonClass): +class TimePeriod(SDMXDimension): def __init__(self): # sdmx-dimension:timePeriod a qb:DimensionProperty, rdf:Property; # rdfs:range rdfs:Resource; @@ -32,42 +32,12 @@ def __init__(self): # rdfs:label "Time Period"@en; # rdfs:comment "The period of time or point in time to which the measured observation refers."@en; # rdfs:isDefinedBy . - super().__init__(entity='DimensionProperty') - self.data = { - "id": "urn:ngsi-ld:DimensionProperty:timePeriod", - "type": "DimensionProperty", - "language": { - "type": "Property", - "value": ["en"] - }, - "label": { - "type": "Property", - "value": { - "en": "Time Period", - } - }, - "description": { - "type": "Property", - "value": { - "en": "The period of time or point in time to which the measured observation refers.", - } - }, - "concept": { - "type": "Relationship", - "object": "urn:ngsi-ld:Concept:timePeriod" - }, - "identifier": { - "type": "Property", - "value": "timePeriod" - }, - "range": { - "type": "Property", - "value": "xsd:string" - }, - "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] - } + super().__init__(entity_id='timePeriod', + label='Time Period', + description='The period of time or point in time to which the measured observation refers.', + concept_id='timePeriod', + identifier='timePeriod', + entity_range='xsd:string') @staticmethod def fix_value(value): diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index c2d9a7b..98e5370 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -57,6 +57,8 @@ from sdmx2jsonld.sdmxconcepts.currencyconcept import CurrencyConcept from sdmx2jsonld.sdmxconcepts.dissorgconcept import DissOrgConcept +from sdmx2jsonld.cube.measuretype import MeasureType + logger = getLogger() @@ -70,7 +72,7 @@ def __init__(self): 'obsStatus', 'confStatus', 'timeFormat', 'timePerCollect', 'decimals', 'title', 'unitMult', 'compilingOrg', 'dataComp', - 'currency', 'dissOrg'] + 'currency', 'dissOrg', 'measureType'] self.data = { "id": str(), @@ -147,7 +149,8 @@ def __init__(self): self.sdmx_dimensions = { "freq": Frequency(), "refArea": RefArea(), - "timePeriod": TimePeriod() + "timePeriod": TimePeriod(), + "measureType": MeasureType() } self.sdmx_attributes = { @@ -183,7 +186,8 @@ def __init__(self): "compilingOrg": CompilingOrgConcept(), "dataComp": DataCompConcept(), "dissOrg": DissOrgConcept(), - "currency": CurrencyConcept() + "currency": CurrencyConcept(), + "measureType": None } self.sdmx_concept_schemas = CogConceptSchema() diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 27fec00..25b058b 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -154,7 +154,8 @@ def create_data(self, entity_type, data, title): logger.error(f'Unexpected entity type, id: {some_new_component.data["idf"]} ' f'type: {some_new_component.data["type"]}') - self.conceptLists.append(some_new_concept) + if some_new_concept is not None: + self.conceptLists.append(some_new_concept) # we need to check that the conceptSchema is not already defined in the structure if some_new_concept_schema not in self.conceptSchemas: From b5afa4457395562be51cac7c397c09da158bb590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 21 Jun 2023 12:49:27 +0200 Subject: [PATCH 57/74] Correction a comment --- sdmx2jsonld/common/classprecedence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py index a912780..803af7c 100644 --- a/sdmx2jsonld/common/classprecedence.py +++ b/sdmx2jsonld/common/classprecedence.py @@ -60,7 +60,7 @@ def precedence(self, data: list) -> str: classes_values = list(map(lambda x: self.get_value(x), data)) # We need to check if all element of the list are the value 250 because could not be possible to have at - # the same time a DimensionProperty and AttributeProperty, this is an ERROR than we need to report. + # the same time a DimensionProperty and AttributeProperty, this is an ERROR therefore we need to report it. result = all(element == 250 for element in classes_values) and len(data) > 1 if result is True: raise ClassPrecedencePropertyError(data) From 6e84d87455e15f8d671eb6deb01a821742d2a9d1 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 16:01:31 +0200 Subject: [PATCH 58/74] Changed some comments in Readme and the dockerfile itself --- docker/Dockerfile | 13 ++++++------- docker/Readme.md | 4 ++-- ngsild/ngsild_connector.py | 24 +++++++++++++++++++++++- tests/common/test_commonclass.py | 3 ++- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d04b844..06947b0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,20 +24,19 @@ RUN pip install --root-user-action=ignore --prefix=$INSTALLATION_PATH -r /requir FROM python:3.11-alpine as final LABEL "maintainer"="FIWARE Foundation e.V." -LABEL "description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." -LABEL "name"="iotagent-turtle" -LABEL "summary"="IoT Agent for the DataQube representation in RDF Turtle format used in statistical environments" +LABEL "description"="An Internet of Things Agent for the RDF Turtle of SDMX DataQube format. This IoT Agent is designed to be a bridge between SDMX statistical metadata representation in RDF Turtle and ETSI NGSI-LD in JSON-LD format representation to be integrated with FIWARE components." +LABEL "name"="iotagent-turtle" +LABEL "summary"="IoT Agent for the DataQube (SDMX) representation in RDF Turtle format used in statistical environments." LABEL "org.opencontainers.image.authors"="fernando.lopez@fiware.org" -LABEL "org.opencontainers.image.documentation"="" +LABEL "org.opencontainers.image.documentation"="https://github.com/flopezag/IoTAgent-Turtle/tree/master/doc" LABEL "org.opencontainers.image.vendor"="FIWARE Foundation e.V." LABEL "org.opencontainers.image.licenses"="Apache2.0" LABEL "org.opencontainers.image.title"="iotagent-turtle" -LABEL "org.opencontainers.image.description"="An Internet of Things Agent for the RDF DataQube Turtle format. This IoT Agent is designed to be a bridge between ISOXML/ADAPT and the ETSI NGSI-LD interface of a FIWARE Context Broker." +LABEL "org.opencontainers.image.description"="An Internet of Things Agent for the RDF Turtle of SDMX DataQube format. This IoT Agent is designed to be a bridge between SDMX statistical metadata representation in RDF Turtle and ETSI NGSI-LD in JSON-LD format representation to be integrated with FIWARE components." LABEL "org.opencontainers.image.source"="https://github.com/flopezag/IoTAgent-Turtle" LABEL "org.opencontainers.image.version"="0.1.0" -LABEL "org.python.version"="python:3.9" - +LABEL "org.python.version"="python:3.11" ARG PROJECT=flopezag ARG COMPONENT=IoTAgent-Turtle diff --git a/docker/Readme.md b/docker/Readme.md index 72287c1..171525e 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -1,7 +1,7 @@ # Docker for server ## build -The docker is easyly build using the following command - By default it builds the docker for the **develop** branch: +The docker is easily built using the following command - By default, it builds the docker for the **develop** branch: ``` docker build . -t iotagent-turtle @@ -46,7 +46,7 @@ docker run --rm -p 5000:5000 --name io -v our-local-config.json:/opt/IoTAgent-tu ### As docker file -We can consider writing a docker-compose.yaml file as the following one to start everything (orion-ld, the name of our orion server is named according to our config.json file (see "broker" key): +We can consider writing a docker-compose.yaml file as the following one to start everything (orion-ld, the name of our orion server is named according to our config.json file -- see "broker" key): ``` version: "3.8" diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index 04c1d57..0124fda 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -1,3 +1,25 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2023 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + from pathlib import Path import json from requests import post, exceptions @@ -31,7 +53,7 @@ def send_data_array(self, json_object): "reason": r}) except TypeError as e: return_info.append({"id": "UNK", - "status_code": 500, + "status_code": 422, "reason": e.args[0]}) except Exception as e: raise e diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py index af5590c..7810a53 100644 --- a/tests/common/test_commonclass.py +++ b/tests/common/test_commonclass.py @@ -56,4 +56,5 @@ def test_save(self): assert(data['id'] == urnid) assert(data['@context'] == context['@context']) - # TODO - Add tests with cclass.generate_id using update_id with a False value \ No newline at end of file + # TODO - Add tests with cclass.generate_id using update_id with a False value + From 4abd42a19040bf20c074d7e1ef493ed7fc60a9fe Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 17:07:37 +0200 Subject: [PATCH 59/74] changed image for imagine --- docker/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Readme.md b/docker/Readme.md index 171525e..93e4520 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -28,7 +28,7 @@ Everything will install in /proc/IotAgent-turtle inside the container. The defau ``` ## Use -Let's suppose the image is named **iotagent-turtle** and we want to expose port 5000 in order to work. We can also image that we want to remove the container when it finishes, we can do it this way: +Let's suppose the image is named **iotagent-turtle** and we want to expose port 5000 in order to work. We can also imagine that we want to remove the container when it finishes, we can do it this way: ``` docker run --rm -p 5000:5000 --name io iotagent-turtle ``` From 43e1e33450d5688e526584c6ccf51e5ea68d062f Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Wed, 21 Jun 2023 17:20:25 +0200 Subject: [PATCH 60/74] changed magic numbers for fastapip.HTTP_XXX values --- ngsild/ngsild_connector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index 0124fda..8a34807 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -23,6 +23,7 @@ from pathlib import Path import json from requests import post, exceptions +from fastapi import status class NGSILDConnector: @@ -53,7 +54,7 @@ def send_data_array(self, json_object): "reason": r}) except TypeError as e: return_info.append({"id": "UNK", - "status_code": 422, + "status_code": status.HTTP_422_UNPROCESSABLE_ENTITY, "reason": e.args[0]}) except Exception as e: raise e @@ -80,7 +81,7 @@ def send_data(self, json_object): # resp = json.loads(r.text) response_status_code = r.status_code - if response_status_code == 201: + if response_status_code == status.HTTP_201_CREATED: print("LOCATION: ", r.headers['Location']) # Let exceptions raise.... They can be controlled somewhere else. From dd087052cf8cd105aa1249f11f25debe1fef4ae9 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 22 Jun 2023 10:55:16 +0200 Subject: [PATCH 61/74] changed broken test --- tests/common/test_listmanagement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index 8c8639b..9efa319 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -10,7 +10,7 @@ def test_get_rest_data(self): data = ['a', ['qb:AttributeProperty'], 'rdfs:label', [['"SDMX attribute COMMENT_OBS"', '@en'], ['"Attribut SDMX "', '@fr']], 'dct:created', [['2022-01-15T06:00:00+00:00']], 'dct:identifier', [['"a3003"']], 'dct:modified', [['2022-01-15T06:30:00+00:00']], 'qb:concept', ['http://bauhaus/concepts/definition/c4303'], 'insee:disseminationStatus', ['http://id.insee.fr/codes/base/statutDiffusion/Prive'], 'insee:validationState', [['"Unpublished"']], 'rdfs:range', ['xsd:string'], 'skos:notation', [['"COMMENT_OBS"']]] not_allowed_keys = ['sliceKey', 'component', 'disseminationStatus', 'validationState', 'notation', 'label', 'codeList', 'concept'] further_process_keys = ['component', 'label'] - expected_res = {'dct:created': '2022-01-15T06:00:00+00:00', 'dct:identifier': 'a3003', 'dct:modified': '2022-01-15T06:30:00+00:00', 'rdfs:range': 'xsd:string'} + expected_res = {'created': '2022-01-15T06:00:00+00:00', 'identifier': 'a3003', 'modified': '2022-01-15T06:30:00+00:00', 'range': 'xsd:string'} res = get_rest_data(data, not_allowed_keys, further_process_keys) assert(expected_res == res) From a927bff1dc660c89aef7ab0fd674e08a9c9b51c5 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 22 Jun 2023 11:16:58 +0200 Subject: [PATCH 62/74] Some property name changed in file ngsild_connector.py and was properly reset --- ngsild/ngsild_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index c539854..eaba31a 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -84,10 +84,10 @@ def send_data(self, json_object): response_status_code = response.status_code if response_status_code == status.HTTP_201_CREATED: - print("LOCATION: ", r.headers['Location']) + print("LOCATION: ", response.headers['Location']) # Let exceptions raise.... They can be controlled somewhere else. - return response_status_code, r.reason + return response_status_code, response.reason if __name__ == "__main__": From 6db565e6cc5f09154b010ea2e2d62ff8f38463be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 22 Jun 2023 11:36:26 +0200 Subject: [PATCH 63/74] Resolution of a bug in Distribution class --- sdmx2jsonld/transform/distribution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index 7d5f71f..d45e939 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -75,7 +75,7 @@ def generate_data(self, catalogue): self.data['title']['value'] = catalogue.data['title']['value'] # language es obtained from language from the Catalogue - self.data['language'] = catalogue.data['language']['value'] + self.data['language']['value'] = catalogue.data['language']['value'] # accessURL is generated from the configuration file. config_path = Path.cwd().joinpath('common/config.json') From e32bd49c488fec72e4f8b9a8c2a72a2df5066221 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 22 Jun 2023 12:23:30 +0200 Subject: [PATCH 64/74] Added logs to ngsild in case of HTTP/400 to visualize the problematic entity --- ngsild/ngsild_connector.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index eaba31a..cfb3a09 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -26,10 +26,13 @@ from fastapi import status from sdmx2jsonld.transform.parser import Parser from io import StringIO +from logging import getLogger + class NGSILDConnector: def __init__(self, path=None): + self.logger = getLogger(__name__) if path is None: config_path = Path.cwd().joinpath('common/config.json') else: @@ -51,7 +54,7 @@ def send_data_array(self, json_object): for elem in d: try: - rc, r = self.send_data(json.dumps(elem)) + rc, r = self.send_data(json.dumps(elem, indent=4)) return_info.append({"id": elem['id'], "status_code": rc, "reason": r}) @@ -84,7 +87,9 @@ def send_data(self, json_object): response_status_code = response.status_code if response_status_code == status.HTTP_201_CREATED: - print("LOCATION: ", response.headers['Location']) + self.logger.debug(f"LOCATION: {response.headers['Location']}") + if response_status_code == status.HTTP_400_BAD_REQUEST: + self.logger.info(f" Parser error: {response.reason}\n{json_object}") # Let exceptions raise.... They can be controlled somewhere else. return response_status_code, response.reason From 2741531e258019937651086a93c0aeca4496e0c4 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 22 Jun 2023 14:31:52 +0200 Subject: [PATCH 65/74] Changed api.yaml file describing the operations and results returned by the server --- doc/api.yaml | 291 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 264 insertions(+), 27 deletions(-) diff --git a/doc/api.yaml b/doc/api.yaml index 76c7757..bff0016 100644 --- a/doc/api.yaml +++ b/doc/api.yaml @@ -32,7 +32,7 @@ paths: post: tags: - Parse - description: Parse a SDMX Turtle file into a NGSI-LD (JSON-LD) format. + description: Parse a SDMX Turtle file into a NGSI-LD (JSON-LD) format and try to create all the entities found in the ngsi-ld Context Broker. operationId: parse requestBody: required: true @@ -42,37 +42,91 @@ paths: $ref: '#/components/schemas/TurtleFile' responses: '201': - description: File succesfully read, parsed, and forwarded to the FIWARE Context Broker. + description: File succesfully read, parsed, and forwarded to the FIWARE Context Broker with several entities. For each of these entities a request to the Context Broker is performed. The result will be a json array containing an explanation to the result of every entity which was processed. The client application will be able to query if everything was properly created in the context broker reading each entry of the result array. content: application/json: schema: - $ref: '#/components/schemas/VersionInfo' + type: array + items: + oneOf: + - $ref: '#/components/schemas/Http201ResponseInfo' + - $ref: '#/components/schemas/Http409ResponseInfo' + - $ref: '#/components/schemas/Http400ResponseInfo' + - $ref: '#/components/schemas/HttpOtherErrorsResponseInfo' '500': - description: Did not receive a valid NGSI-LD Entity. - content: - application/json: - schema: - $ref: '#/components/schemas/VersionInfo' - '400': - description: Did not receive a valid NGSI-LD Entity. - content: - application/json: - schema: - $ref: '#/components/schemas/VersionInfo' - '408': - description: Did not receive a valid NGSI-LD Entity. - content: - application/json: - schema: - $ref: '#/components/schemas/VersionInfo' - '503': - description: Did not receive a valid NGSI-LD Entity. + description: Problem connecting or accessing Context Broker. content: application/json: schema: - $ref: '#/components/schemas/VersionInfo' + $ref: '#/components/schemas/Http500Error' components: schemas: + Http201ResponseInfo: + type: object + description: Return status from CB to a given entity creation request + properties: + id: + type: string + example: "urn:ngsi-ld:Distribution:d2c32c6c6e637c0b319139ff25c1eff4" + status_code: + type: number + example: 201 + description: The entity was created in the CB. + reason: + type: string + example: "Created" + Http400ResponseInfo: + type: object + description: Return status from CB to a given entity creation request + properties: + id: + type: string + example: "urn:ngsi-ld:AttributeProperty:a3018" + status_code: + type: number + example: 400 + description: The format of the entity is not appropriate to the context broker, thus it is not created and HTTP/400 is returned. + reason: + type: string + example: "Bad Request" + Http409ResponseInfo: + type: object + description: Return status from CB to a given entity creation request + properties: + id: + type: string + example: "urn:ngsi-ld:Observation:obs-A-N-BE-W2-S1-S1-NA-B1G-_Z-A-_Z-XDC-V-N-2013" + status_code: + type: number + example: 409 + description: The entity can't be created because it already exists in the Contest Broker. + reason: + type: string + example: "Conflict" + description: Return reason for an entity sent to the Context Broker for creation. Likely values are "Created", "Bad Request", "Conflict", "Unprocessable entity" depending and acordingly to the previous status_code value. + HttpOtherErrorsResponseInfo: + type: object + description: Return status from CB to a given entity creation request + properties: + id: + type: string + example: "urn:ngsi-ld:Concept:dissOrg" + status_code: + type: number + example: 422 + description: The entity tryed to be creted in the context Broker shows other errors.. + reason: + type: string + example: "Unprocessable entity" + description: Return reason will describe the problem with the creation of the entity in the context broker. It will be different to the previously explained ones and to the 500 error. + Http500Error: + type: object + description: Detailed error / problem with Context Broker + properties: + detail: + type: string + example: | + HTTPConnectionPool(host='orion-ld', port=1026): Max retries exceeded with url: /ngsi-ld/v1/entities (Caused by NameResolutionError(\": Failed to resolve 'orion-ld' ([Errno -5] Name has no usable address)\")) VersionInfo: type: object description: Status of the running service @@ -87,7 +141,7 @@ components: example: "UP" version: type: string - description: Current version of the IoTAgent-Turtle server. + description: Current version of the IoTAgent-Turtle server example: "UP" release_date: type: string @@ -108,7 +162,190 @@ components: type: string format: binary description: | - This is a string - in multiple lines. + @prefix rdf: . + + @prefix dc: . + + @prefix dcterms: . + + @prefix qb: . + + @prefix rdfs: . + + @prefix owl: . + + @prefix skos: . + + @prefix xsd: . + + @prefix sdmx: . + + @prefix sdmx-concept: . + + @prefix sdmx-dimension: . + + @prefix sdmx-attribute: . + + @prefix sdmx-measure: . + + @prefix sdmx-metadata: . + + @prefix sdmx-code: . + + @prefix sdmx-subject: . + + + + a qb:DataSet ; + + dcterms:issued "2022-04-01T08:00:00.000"^^xsd:dateTime ; + + dcterms:publisher ; + + dcterms:title "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr ; + + qb:structure ; + + rdfs:label "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr ; + + sdmx-attribute:title "GDP and main components (current prices)"@en, "PIB et principales composantes (prix courants)"@fr . + + + + a qb:Observation; + + ; + + "W2" ; + + "S1" ; + + "S1" ; + + "B" ; + + "B1G" ; + + "_Z" ; + + "A" ; + + "_Z" ; + + "XDC" ; + + "V" ; + + "N" ; + + qb:dataSet ; + + sdmx-attribute:confStatus sdmx-code:confStatus-F ; + + sdmx-attribute:decimals sdmx-code:decimals-1 ; + + sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; + + sdmx-attribute:unitMult sdmx-code:unitMult-6 ; + + sdmx-dimension:freq sdmx-code:freq-A ; + + sdmx-dimension:refArea "BE" ; + + sdmx-dimension:timePeriod "2011" ; + + sdmx-measure:obsValue "1016.9"^^xsd:float . + + + + a qb:Observation; + + ; + + "W2" ; + + "S1" ; + + "S1" ; + + "B" ; + + "B1G" ; + + "_Z" ; + + "A" ; + + "_Z" ; + + "XDC" ; + + "V" ; + + "N" ; + + qb:dataSet ; + + sdmx-attribute:confStatus sdmx-code:confStatus-F ; + + sdmx-attribute:decimals sdmx-code:decimals-1 ; + + sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; + + sdmx-attribute:unitMult sdmx-code:unitMult-6 ; + + sdmx-dimension:freq sdmx-code:freq-A ; + + sdmx-dimension:refArea "BE" ; + + sdmx-dimension:timePeriod "2012" ; + + sdmx-measure:obsValue "3016.9"^^xsd:float . + + + + a qb:Observation; + + ; + + "W2" ; + + "S1" ; + + "S1" ; + + "B" ; + + "B1G" ; + + "_Z" ; + + "A" ; + + "_Z" ; + + "XDC" ; + + "V" ; + + "N" ; + + qb:dataSet ; + + sdmx-attribute:confStatus sdmx-code:confStatus-F ; + + sdmx-attribute:decimals sdmx-code:decimals-1 ; + + sdmx-attribute:obsStatus sdmx-code:obsStatus-A ; + + sdmx-attribute:unitMult sdmx-code:unitMult-6 ; + + sdmx-dimension:freq sdmx-code:freq-A ; + + sdmx-dimension:refArea "BE" ; + + sdmx-dimension:timePeriod "2013" ; + + sdmx-measure:obsValue "9016.9"^^xsd:float . + - And an extra one. From 45b1414f313e095b70100b9a5684e9325707c838 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Carretero Date: Thu, 22 Jun 2023 14:38:01 +0200 Subject: [PATCH 66/74] Removed on line. It was incorrect --- doc/api.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/api.yaml b/doc/api.yaml index bff0016..77623fb 100644 --- a/doc/api.yaml +++ b/doc/api.yaml @@ -103,7 +103,6 @@ components: reason: type: string example: "Conflict" - description: Return reason for an entity sent to the Context Broker for creation. Likely values are "Created", "Bad Request", "Conflict", "Unprocessable entity" depending and acordingly to the previous status_code value. HttpOtherErrorsResponseInfo: type: object description: Return status from CB to a given entity creation request From 3cdafa360c72b08fc10bbae1febf3d73d5512d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 22 Jun 2023 16:29:10 +0200 Subject: [PATCH 67/74] Review bug #132 --- sdmx2jsonld/transform/entitytype.py | 2 +- sdmx2jsonld/transform/parser.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 25b058b..7b8d225 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -151,7 +151,7 @@ def create_data(self, entity_type, data, title): self.attributes.append(some_new_component) else: # You should not be here, reporting error... - logger.error(f'Unexpected entity type, id: {some_new_component.data["idf"]} ' + logger.error(f'Unexpected entity type, id: {some_new_component.data["id"]} ' f'type: {some_new_component.data["type"]}') if some_new_concept is not None: diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index c7bdd0e..2c4e3c5 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -67,6 +67,8 @@ def parsing_file(self, content: TextIOWrapper, out: bool): data = f.read() data = turtle_terse(rdf_content=data) + with open("./logs/final.ttl", "w") as outfile: + outfile.write(data) try: tree = self.parser.parse(data) @@ -128,7 +130,6 @@ def parsing_string(self, content: StringIO): dimensions = transform.get_dimensions() [result.append(x.get()) for x in dimensions] - attr = transform.get_attributes() [result.append(x.get()) for x in transform.get_attributes()] [result.append(x.get()) for x in transform.get_concept_schemas()] From aa832e94d6c5fcd0e8edcbb1c8cff00b995c6e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 23 Jun 2023 12:50:02 +0200 Subject: [PATCH 68/74] Review testing process --- .gitignore | 3 + ngsild/ngsild_connector.py | 2 +- sdmx2jsonld/transform/distribution.py | 5 +- tests/common/test_listmanagement.py | 22 ++++ tests/common/test_rdf.py | 22 ++++ tests/common/test_regparser.py | 23 ++++ .../files/observations.ttl | 0 .../files}/structures-accounts.ttl | 0 .../files}/structures-tourism.ttl | 0 tests/sdmxattributes/test_code.py | 21 ++++ .../sdmxattributes/test_confirmationStatus.py | 21 ++++ tests/sdmxattributes/test_examples.py | 103 ++++++++++++++---- .../sdmxattributes/test_observationStatus.py | 21 ++++ 13 files changed, 221 insertions(+), 22 deletions(-) rename examples/observation.ttl => tests/files/observations.ttl (100%) rename {examples => tests/files}/structures-accounts.ttl (100%) rename {examples => tests/files}/structures-tourism.ttl (100%) diff --git a/.gitignore b/.gitignore index cedcb13..53796fa 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ dmypy.json output output/*.jsonld tests/test1.py + +# logs folder +logs diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index cfb3a09..b300387 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -100,7 +100,7 @@ def send_data(self, json_object): print(c.get_url()) parser = Parser() - with open("../examples/structures-tourism.ttl", "r") as rf: + with open("../tests/files/structures-tourism.ttl", "r") as rf: rdf_data = rf.read() r = parser.parsing(StringIO(rdf_data), out=False) diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index d45e939..64a5a2c 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -21,9 +21,9 @@ ## from logging import getLogger from sdmx2jsonld.common.commonclass import CommonClass -from pathlib import Path from json import load from random import getrandbits +from os.path import dirname, join logger = getLogger() @@ -78,7 +78,8 @@ def generate_data(self, catalogue): self.data['language']['value'] = catalogue.data['language']['value'] # accessURL is generated from the configuration file. - config_path = Path.cwd().joinpath('common/config.json') + config_path = dirname(dirname(dirname(__file__))) + config_path = join(join(config_path, 'common'), 'config.json') with open(config_path) as config_file: config = load(config_file) diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index 9efa319..46bd2c9 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -1,7 +1,29 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value, extract_prefix, get_property_value from sdmx2jsonld.exceptions.exceptions import ClassExtractPrefixError + class TestRegToParser(TestCase): def setUp(self) -> None: pass diff --git a/tests/common/test_rdf.py b/tests/common/test_rdf.py index 507cb06..12aa53c 100644 --- a/tests/common/test_rdf.py +++ b/tests/common/test_rdf.py @@ -1,7 +1,29 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.common.rdf import turtle_terse from rdflib import Graph + class TestRegToParser(TestCase): def setUp(self) -> None: pass diff --git a/tests/common/test_regparser.py b/tests/common/test_regparser.py index 851ef94..5dabb9c 100644 --- a/tests/common/test_regparser.py +++ b/tests/common/test_regparser.py @@ -1,5 +1,28 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.common.regparser import RegParser + + class TestRegToParser(TestCase): def setUp(self) -> None: pass diff --git a/examples/observation.ttl b/tests/files/observations.ttl similarity index 100% rename from examples/observation.ttl rename to tests/files/observations.ttl diff --git a/examples/structures-accounts.ttl b/tests/files/structures-accounts.ttl similarity index 100% rename from examples/structures-accounts.ttl rename to tests/files/structures-accounts.ttl diff --git a/examples/structures-tourism.ttl b/tests/files/structures-tourism.ttl similarity index 100% rename from examples/structures-tourism.ttl rename to tests/files/structures-tourism.ttl diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py index 4ff4b54..05adc50 100644 --- a/tests/sdmxattributes/test_code.py +++ b/tests/sdmxattributes/test_code.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.sdmxattributes.code import Code from sdmx2jsonld.exceptions.exceptions import ClassCode diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py index d8fcb87..806abdb 100644 --- a/tests/sdmxattributes/test_confirmationStatus.py +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.sdmxattributes.confirmationStatus import ConfStatus from sdmx2jsonld.exceptions.exceptions import ClassConfStatusError diff --git a/tests/sdmxattributes/test_examples.py b/tests/sdmxattributes/test_examples.py index ed34128..1130025 100644 --- a/tests/sdmxattributes/test_examples.py +++ b/tests/sdmxattributes/test_examples.py @@ -1,38 +1,103 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.transform.parser import Parser from io import StringIO -import os, shutil +from os import listdir, unlink +from os.path import join, isfile, dirname +from pathlib import Path + class TestCommonClass(TestCase): def setUp(self) -> None: - pass + examples_folder = dirname(dirname(__file__)) + output_folder = dirname(examples_folder) + + examples_folder = join(examples_folder, 'files') + self.output_folder = join(output_folder, 'output') + + tests_files = ["observations.ttl", "structures-accounts.ttl", "structures-tourism.ttl"] + self.tests_files = [join(examples_folder, x) for x in tests_files] + + self.parser = Parser() def clean_output_dir(self) -> None: - for a in os.listdir("output"): - file_path = os.path.join("output", a) + for a in listdir(self.output_folder): + file_path = join(self.output_folder, a) try: - if os.path.isfile(file_path): - os.unlink(file_path) + if isfile(file_path): + unlink(file_path) except Exception as e: print("----------------------------") print(e.message) print("----------------------------") - def test_file_1(self): - for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: + def test_files_from_StringIO_web_interface_loop(self): + print("Testing test_files_from_StringIO_web_interface_loop") + for a in self.tests_files: + print(f"Parsing: {a}") + self.clean_output_dir() + + # Read the RDF Turtle file + with open(a, "r") as rf: + rdf_data = rf.read() + + # Parsing the RDF + _ = self.parser.parsing(content=StringIO(rdf_data), out=False) + + print("Parsing completed...\n") + + print("Test finished...\n") + + def test_files_from_TextIOWrapper_cli_with_generating_files(self): + print("Testing test_files_from_TextIOWrapper_cli_with_generating_files") + for a in self.tests_files: print(f"Parsing: {a}") - parser = Parser() - with open(f"../../examples/{a}", "r") as rf: + self.clean_output_dir() + + # Read the RDF Turtle file + with open(a, "r") as rf: rdf_data = rf.read() - r = parser.parsing(content=StringIO(rdf_data), out=False) - def test_file_2(self): - for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: + # Parsing the RDF + _ = self.parser.parsing(content=rdf_data, out=True) + + print("Parsing completed...\n") + + print("Test finished...\n") + + def test_file_from_TextIOWrapper_cli_only_printing_result(self): + print("Testing test_files_from_TextIOWrapper_cli_with_generating_files") + for a in self.tests_files: + print(f"Parsing: {a}") self.clean_output_dir() - parser = Parser() - r = parser.parsing(content=open(f"../../examples/{a}", "r"), out=True) - def test_file_3(self): - for a in ["observation.ttl", "structures-accounts.ttl", "structures-tourism.ttl"]: - parser = Parser() - r = parser.parsing(content=open(f"../../examples/{a}", "r"), out=False) + # Read the RDF Turtle file + with open(a, "r") as rf: + rdf_data = rf.read() + + # Parsing the RDF + _ = self.parser.parsing(content=rdf_data, out=False) + + print("Parsing completed...\n") + + print("Test finished...\n") diff --git a/tests/sdmxattributes/test_observationStatus.py b/tests/sdmxattributes/test_observationStatus.py index 59495e6..b7f074d 100644 --- a/tests/sdmxattributes/test_observationStatus.py +++ b/tests/sdmxattributes/test_observationStatus.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +## +# Copyright 2022 FIWARE Foundation, e.V. +# +# This file is part of IoTAgent-SDMX (RDF Turtle) +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## from unittest import TestCase from sdmx2jsonld.sdmxattributes.observationStatus import ObsStatus from sdmx2jsonld.exceptions.exceptions import ClassObsStatusError From 867956b1626b9ce8b4730e44c0ac54e780b790b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Wed, 28 Jun 2023 15:20:21 +0200 Subject: [PATCH 69/74] Complete test_examples execution tests --- tests/sdmxattributes/test_examples.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/sdmxattributes/test_examples.py b/tests/sdmxattributes/test_examples.py index 1130025..2160fd5 100644 --- a/tests/sdmxattributes/test_examples.py +++ b/tests/sdmxattributes/test_examples.py @@ -62,7 +62,12 @@ def test_files_from_StringIO_web_interface_loop(self): rdf_data = rf.read() # Parsing the RDF - _ = self.parser.parsing(content=StringIO(rdf_data), out=False) + try: + _ = self.parser.parsing(content=StringIO(rdf_data), out=False) + except Exception as e: + assert False, f"\nThe parser was not completed," \ + f"\n file: {a}" \ + f"\n exception:\n {e.message}" print("Parsing completed...\n") @@ -79,7 +84,12 @@ def test_files_from_TextIOWrapper_cli_with_generating_files(self): rdf_data = rf.read() # Parsing the RDF - _ = self.parser.parsing(content=rdf_data, out=True) + try: + _ = self.parser.parsing(content=rdf_data, out=True) + except Exception as e: + assert False, f"\nThe parser was not completed," \ + f"\n file: {a}" \ + f"\n exception:\n {e.message}" print("Parsing completed...\n") @@ -96,7 +106,12 @@ def test_file_from_TextIOWrapper_cli_only_printing_result(self): rdf_data = rf.read() # Parsing the RDF - _ = self.parser.parsing(content=rdf_data, out=False) + try: + _ = self.parser.parsing(content=rdf_data, out=False) + except Exception as e: + assert False, f"\nThe parser was not completed," \ + f"\n file: {a}" \ + f"\n exception:\n {e.message}" print("Parsing completed...\n") From c40ae0f617e507ba49f12aeb84b897ce5dcb7df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 24 Jul 2023 13:54:29 +0200 Subject: [PATCH 70/74] Update requirements.txt and resolve an issue in Concept Schema when dct:created and dct:modified is not present in Turtle file --- requirements.txt | 13 ++++++------- sdmx2jsonld/transform/conceptschema.py | 26 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index 363db7e..21a0aa2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ -# python3.10, virtualenv -ppython3.10 .venv -lark==1.1.5 +# python3.11, virtualenv -ppython3.11 .venv +lark==1.1.7 secure==0.3.0 docopt==0.6.2 schema==0.7.5 hi-dateinfer==0.4.6 -fastapi==0.97.0 -uvicorn==0.22.0 +fastapi==0.100.0 +uvicorn==0.23.1 python-multipart==0.0.6 loguru==0.7.0 requests==2.31.0 -rdflib>=6.2,<6.4 -python-dateutil>=2.8.2 -pytz>=2022.6 +rdflib==6.3.2 +python-dateutil==2.8.2 diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 4ff58d0..0ca4b32 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -115,17 +115,23 @@ def add_data(self, concept_schema_id, data): self.data['hasTopConcept']['object'] = result # Get the rest of data, dct:created and dct:modified properties - position = data.index('dct:created') + 1 - self.data['created'] = { - "type": "Property", - "value": flatten_value(data[position]) - } + try: + position = data.index('dct:created') + 1 + self.data['created'] = { + "type": "Property", + "value": flatten_value(data[position]) + } + except ValueError: + logger.warning(f'dct:created is not present in the Concept Schema: {concept_schema_id}') - position = data.index('dct:modified') + 1 - self.data['modified'] = { - "type": "Property", - "value": flatten_value(data[position]) - } + try: + position = data.index('dct:modified') + 1 + self.data['modified'] = { + "type": "Property", + "value": flatten_value(data[position]) + } + except ValueError: + logger.warning(f'dct:modified is not present in the Concept Schema: {concept_schema_id}') # Order the keys in the final json-ld a = Context() From 7729064c6d3748b69a9827a8eec05260376af078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 25 Jul 2023 08:34:24 +0200 Subject: [PATCH 71/74] Linter and type checking --- agent.py | 18 +- api/custom_logging.py | 55 +- api/server.py | 125 ++-- cli/command.py | 33 +- ngsild/ngsild_connector.py | 23 +- poetry.lock | 689 +++++++++++------- pyproject.toml | 30 +- requirements-dev.txt | 8 + sdmx2jsonld/common/classprecedence.py | 31 +- sdmx2jsonld/common/commonclass.py | 24 +- sdmx2jsonld/common/config.py | 12 +- sdmx2jsonld/common/datatypeconversion.py | 97 ++- sdmx2jsonld/common/listmanagement.py | 34 +- sdmx2jsonld/common/regparser.py | 6 +- sdmx2jsonld/cube/measuretype.py | 18 +- sdmx2jsonld/exceptions/__init__.py | 33 +- sdmx2jsonld/exceptions/exceptions.py | 7 +- sdmx2jsonld/sdmxattributes/code.py | 15 +- sdmx2jsonld/sdmxattributes/compilingorg.py | 14 +- .../sdmxattributes/confirmationStatus.py | 41 +- sdmx2jsonld/sdmxattributes/currency.py | 14 +- sdmx2jsonld/sdmxattributes/dataComp.py | 16 +- sdmx2jsonld/sdmxattributes/decimals.py | 14 +- sdmx2jsonld/sdmxattributes/dissorg.py | 14 +- .../sdmxattributes/observationStatus.py | 26 +- sdmx2jsonld/sdmxattributes/sdmxattribute.py | 29 +- sdmx2jsonld/sdmxattributes/timeFormat.py | 14 +- sdmx2jsonld/sdmxattributes/timePerCollect.py | 18 +- sdmx2jsonld/sdmxattributes/title.py | 14 +- sdmx2jsonld/sdmxattributes/unitmult.py | 16 +- sdmx2jsonld/sdmxconcepts/cogconceptschema.py | 13 +- .../sdmxconcepts/compilingorgconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/currencyconcept.py | 9 +- sdmx2jsonld/sdmxconcepts/datacompconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/decimals.py | 10 +- sdmx2jsonld/sdmxconcepts/dissorgconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/freqconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/sdmxconcept.py | 23 +- .../sdmxconcepts/timePerCollectConcept.py | 10 +- sdmx2jsonld/sdmxconcepts/timeformatconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 10 +- sdmx2jsonld/sdmxconcepts/titleConcept.py | 10 +- sdmx2jsonld/sdmxconcepts/unitmultconcept.py | 10 +- sdmx2jsonld/sdmxdimensions/frequency.py | 16 +- sdmx2jsonld/sdmxdimensions/refarea.py | 16 +- sdmx2jsonld/sdmxdimensions/sdmxdimension.py | 41 +- sdmx2jsonld/sdmxdimensions/timeperiod.py | 16 +- sdmx2jsonld/transform/attribute.py | 6 +- sdmx2jsonld/transform/catalogue.py | 96 ++- sdmx2jsonld/transform/concept.py | 98 ++- sdmx2jsonld/transform/conceptschema.py | 75 +- sdmx2jsonld/transform/context.py | 113 +-- sdmx2jsonld/transform/dataset.py | 220 +++--- sdmx2jsonld/transform/dimension.py | 6 +- sdmx2jsonld/transform/distribution.py | 38 +- sdmx2jsonld/transform/entitytype.py | 96 ++- sdmx2jsonld/transform/observation.py | 150 ++-- sdmx2jsonld/transform/parser.py | 28 +- sdmx2jsonld/transform/property.py | 98 ++- sdmx2jsonld/transform/transformer.py | 12 +- tests/common/test_classprecedence.py | 135 ++-- tests/common/test_commonclass.py | 34 +- tests/common/test_datatypeconversion.py | 153 ++-- tests/common/test_listmanagement.py | 237 +++--- tests/common/test_rdf.py | 8 +- tests/common/test_regparser.py | 4 +- tests/sdmxattributes/test_code.py | 68 +- .../sdmxattributes/test_confirmationStatus.py | 70 +- tests/sdmxattributes/test_examples.py | 38 +- .../sdmxattributes/test_observationStatus.py | 72 +- tox.ini | 38 +- 74 files changed, 2086 insertions(+), 1559 deletions(-) create mode 100644 requirements-dev.txt diff --git a/agent.py b/agent.py index fb22fdc..55a5de3 100644 --- a/agent.py +++ b/agent.py @@ -26,12 +26,12 @@ from sdmx2jsonld.exceptions import UnexpectedEOF, UnexpectedInput, UnexpectedToken from ngsild.ngsild_connector import NGSILDConnector -if __name__ == '__main__': +if __name__ == "__main__": args = parse_cli() - if args['run'] is True: - file_in = args['--input'] - generate_files = args['--output'] + if args["run"] is True: + file_in = args["--input"] + generate_files = args["--output"] my_parser = Parser() @@ -44,10 +44,8 @@ except UnexpectedEOF as e: print(e) - elif args['server'] is True: - port = int(args['--port']) - host = args['--host'] + elif args["server"] is True: + port = int(args["--port"]) + host = args["--host"] - launch(app="api.server:application", - host=host, - port=port) + launch(app="api.server:application", host=host, port=port) diff --git a/api/custom_logging.py b/api/custom_logging.py index 4f35224..289f779 100644 --- a/api/custom_logging.py +++ b/api/custom_logging.py @@ -30,12 +30,12 @@ class InterceptHandler(Handler): loglevel_mapping = { - 50: 'CRITICAL', - 40: 'ERROR', - 30: 'WARNING', - 20: 'INFO', - 10: 'DEBUG', - 0: 'NOTSET', + 50: "CRITICAL", + 40: "ERROR", + 30: "WARNING", + 20: "INFO", + 10: "DEBUG", + 0: "NOTSET", } def emit(self, record): @@ -49,44 +49,36 @@ def emit(self, record): frame = frame.f_back depth += 1 - log = logger.bind(request_id='app') - log.opt( - depth=depth, - exception=record.exc_info - ).log(level, record.getMessage()) + log = logger.bind(request_id="app") + log.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage()) class CustomizeLogger: @classmethod def make_logger(cls, config_path: Path): config = cls.load_logging_config(config_path) - logging_config = config.get('logger') + logging_config = config.get("logger") logger = cls.customize_logging( - filepath=logging_config.get('path'), - level=logging_config.get('level'), - retention=logging_config.get('retention'), - rotation=logging_config.get('rotation'), - format=logging_config.get('format')) + filepath=logging_config.get("path"), + level=logging_config.get("level"), + retention=logging_config.get("retention"), + rotation=logging_config.get("rotation"), + format=logging_config.get("format"), + ) return logger @classmethod - def customize_logging(cls, - filepath: Path, - level: str, - rotation: str, - retention: str, - format: str): + def customize_logging( + cls, filepath: Path, level: str, rotation: str, retention: str, format: str + ): logger.remove() logger.add( - sys.stdout, - enqueue=True, - backtrace=True, - level=level.upper(), - format=format) + sys.stdout, enqueue=True, backtrace=True, level=level.upper(), format=format + ) logger.add( str(filepath), @@ -95,12 +87,13 @@ def customize_logging(cls, enqueue=True, backtrace=True, level=level.upper(), - format=format) + format=format, + ) basicConfig(handlers=[InterceptHandler()], level=0) getLogger("uvicorn.access").handlers = [InterceptHandler()] - for _log in ['uvicorn', 'uvicorn.error', 'uvicorn.access', 'fastapi']: + for _log in ["uvicorn", "uvicorn.error", "uvicorn.access", "fastapi"]: _logger = getLogger(_log) _logger.handlers = [InterceptHandler()] @@ -111,4 +104,4 @@ def load_logging_config(cls, config_path): config = None with open(config_path) as config_file: config = load(config_file) - return config \ No newline at end of file + return config diff --git a/api/server.py b/api/server.py index e1e6346..f02cbaa 100644 --- a/api/server.py +++ b/api/server.py @@ -21,13 +21,21 @@ ## from fastapi import FastAPI, UploadFile, Request, Response, status, HTTPException +from fastapi.logger import logger as fastapi_logger from uvicorn import run from os.path import splitext from sdmx2jsonld.transform.parser import Parser from datetime import datetime from cli.command import __version__ -from secure import Server, ContentSecurityPolicy, StrictTransportSecurity, \ - ReferrerPolicy, PermissionsPolicy, CacheControl, Secure +from secure import ( + Server, + ContentSecurityPolicy, + StrictTransportSecurity, + ReferrerPolicy, + PermissionsPolicy, + CacheControl, + Secure, +) from logging import getLogger from pathlib import Path from api.custom_logging import CustomizeLogger @@ -42,10 +50,11 @@ def create_app() -> FastAPI: - app = FastAPI(title='IoTAgent-Turtle', debug=False) - logging_config_path = Path.cwd().joinpath('common/config.json') + app = FastAPI(title="IoTAgent-Turtle", debug=False) + + logging_config_path = Path.cwd().joinpath("common/config.json") customize_logger = CustomizeLogger.make_logger(logging_config_path) - app.logger = customize_logger + fastapi_logger.addHandler(customize_logger) return app @@ -59,20 +68,19 @@ async def set_secure_headers(request, call_next): server = Server().set("Secure") csp = ( - ContentSecurityPolicy().default_src("'none'") - .base_uri("'self'") - .connect_src("'self'" "api.spam.com") - .frame_src("'none'") - .img_src("'self'", "static.spam.com") + ContentSecurityPolicy() + .default_src("'none'") + .base_uri("'self'") + .connect_src("'self'" "api.spam.com") + .frame_src("'none'") + .img_src("'self'", "static.spam.com") ) hsts = StrictTransportSecurity().include_subdomains().preload().max_age(2592000) referrer = ReferrerPolicy().no_referrer() - permissions_value = ( - PermissionsPolicy().geolocation("self", "'spam.com'").vibrate() - ) + permissions_value = PermissionsPolicy().geolocation("self", "'spam.com'").vibrate() cache_value = CacheControl().must_revalidate() @@ -98,7 +106,7 @@ def getversion(request: Request): "git_hash": "nogitversion", "version": __version__, "release_date": "no released", - "uptime": get_uptime() + "uptime": get_uptime(), } return data @@ -109,76 +117,100 @@ async def parse(request: Request, file: UploadFile, response: Response): request.app.logger.info(f'Request parse file "{file.filename}"') # check if the post request has the file part - if splitext(file.filename)[1] != '.ttl': - resp = {'message': 'Allowed file type is only ttl'} + if splitext(file.filename)[1] != ".ttl": # type: ignore[type-var] + resp = {"message": "Allowed file type is only ttl"} response.status_code = status.HTTP_400_BAD_REQUEST - request.app.logger.error(f'POST /parse 400 Bad Request, file: "{file.filename}"') + request.app.logger.error( + f'POST /parse 400 Bad Request, file: "{file.filename}"' + ) else: try: content = await file.read() except Exception as e: - request.app.logger.error(f'POST /parse 500 Problem reading file: "{file.filename}"') - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + request.app.logger.error( + f'POST /parse 500 Problem reading file: "{file.filename}"' + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) else: - request.app.logger.info('File successfully read') + request.app.logger.info("File successfully read") # Prepare the content - content = content.decode("utf-8") + content = content.decode("utf-8") # type: ignore[assignment] # Start parsing the file my_parser = Parser() try: - json_object = my_parser.parsing(content=StringIO(content)) + json_object = my_parser.parsing(content=StringIO(content)) # type: ignore[arg-type] except UnexpectedToken as e: request.app.logger.error(e) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) except UnexpectedInput as e: request.app.logger.error(e) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) except UnexpectedEOF as e: request.app.logger.error(e) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) except Exception as e: - request.app.logger.error(f'POST /parse 500 Problem parsing file: "{file.filename}"') - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + request.app.logger.error( + f'POST /parse 500 Problem parsing file: "{file.filename}"' + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) else: - request.app.logger.info(f'File successfully parsed') + request.app.logger.info(f"File successfully parsed") # Send the data to a FIWARE Context Broker instance headers = { - 'Content-Type': 'application/ld+json', + "Content-Type": "application/ld+json", # 'Accept': 'application/ld+json' } url = get_url() try: - request.app.logger.debug(f'Sending data:\n{json_object}') + request.app.logger.debug(f"Sending data:\n{json_object}") cb = NGSILDConnector() resp = cb.send_data_array(json_object) # resp = loads(r.text) # response.status_code = r.status_code except exceptions.Timeout as err: - request.app.logger.error('Timeout requesting FIWARE Context Broker') - raise HTTPException(status_code=status.HTTP_408_REQUEST_TIMEOUT, detail=str(err)) + request.app.logger.error("Timeout requesting FIWARE Context Broker") + raise HTTPException( + status_code=status.HTTP_408_REQUEST_TIMEOUT, detail=str(err) + ) except exceptions.ConnectionError as err: - message = f'There was a problem connecting to the FIWARE Context Broker. URL: {url}' + message = f"There was a problem connecting to the FIWARE Context Broker. URL: {url}" request.app.logger.error(message) - raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(err)) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(err) + ) except exceptions.HTTPError as e: - request.app.logger.error(f'Call to FIWARE Context Broker failed: {e}') + request.app.logger.error(f"Call to FIWARE Context Broker failed: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except KeyboardInterrupt: - request.app.logger.warning('Server interrupted by user') + request.app.logger.warning("Server interrupted by user") raise except Exception as e: - r = getattr(e, 'message', str(e)) + r = getattr(e, "message", str(e)) request.app.logger.error(r) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(r)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(r) + ) else: - request.app.logger.info(f'Content sent to the Context Broker') - request.app.logger.debug(f'Status Code: {response.status_code}, Response:\n{resp}') + request.app.logger.info(f"Content sent to the Context Broker") + request.app.logger.debug( + f"Status Code: {response.status_code}, Response:\n{resp}" + ) return resp @@ -191,13 +223,13 @@ def get_uptime(): minutes, seconds = divmod(remainder, 60) days, hours = divmod(hours, 24) - fmt = '{d} days, {h} hours, {m} minutes, and {s} seconds' + fmt = "{d} days, {h} hours, {m} minutes, and {s} seconds" return fmt.format(d=days, h=hours, m=minutes, s=seconds) def get_url(): - config_path = Path.cwd().joinpath('common/config.json') + config_path = Path.cwd().joinpath("common/config.json") with open(config_path) as config_file: config = load(config_file) @@ -208,7 +240,14 @@ def get_url(): def launch(app: str = "server:application", host: str = "127.0.0.1", port: int = 5000): - run(app=app, host=host, port=port, log_level="info", reload=True, server_header=False) + run( + app=app, + host=host, + port=port, + log_level="info", + reload=True, + server_header=False, + ) if __name__ == "__main__": diff --git a/cli/command.py b/cli/command.py index b7fd47f..191064e 100644 --- a/cli/command.py +++ b/cli/command.py @@ -47,32 +47,37 @@ from docopt import docopt from os.path import basename from sys import argv -from schema import Schema, And, Or, Use, SchemaError +from schema import Schema, And, Or, Use, SchemaError # type: ignore -__version__ = "0.5.2" +__version__ = "1.0.0" __author__ = "fla" def parse_cli() -> dict: if len(argv) == 1: - argv.append('-h') + argv.append("-h") - version = f'IoTAgent-Turtle version {__version__}' + version = f"IoTAgent-Turtle version {__version__}" args = docopt(__doc__.format(proc=basename(argv[0])), version=version) schema = Schema( { - '--help': bool, - '--input': Or(None, Use(open, error='--input FILE, FILE should be readable')), - '--output': bool, - '--port': Or(None, And(Use(int), lambda n: 1 < n < 65535), - error='--port N, N should be integer 1 < N < 65535'), - '--host': Or(None, str, error='--host HOST should be a string'), - '--version': bool, - 'run': bool, - 'server': bool + "--help": bool, + "--input": Or( + None, Use(open, error="--input FILE, FILE should be readable") + ), + "--output": bool, + "--port": Or( + None, + And(Use(int), lambda n: 1 < n < 65535), + error="--port N, N should be integer 1 < N < 65535", + ), + "--host": Or(None, str, error="--host HOST should be a string"), + "--version": bool, + "run": bool, + "server": bool, } ) @@ -84,5 +89,5 @@ def parse_cli() -> dict: return args -if __name__ == '__main__': +if __name__ == "__main__": print(parse_cli()) diff --git a/ngsild/ngsild_connector.py b/ngsild/ngsild_connector.py index b300387..51b03ab 100644 --- a/ngsild/ngsild_connector.py +++ b/ngsild/ngsild_connector.py @@ -29,19 +29,18 @@ from logging import getLogger - class NGSILDConnector: def __init__(self, path=None): self.logger = getLogger(__name__) if path is None: - config_path = Path.cwd().joinpath('common/config.json') + config_path = Path.cwd().joinpath("common/config.json") else: config_path = Path.cwd().joinpath(path) with open(config_path) as config_file: config = json.load(config_file) - self.base_url = config['broker'] + self.base_url = config["broker"] def get_url(self): url = f"{self.base_url}/ngsi-ld/v1/entities" @@ -55,13 +54,15 @@ def send_data_array(self, json_object): for elem in d: try: rc, r = self.send_data(json.dumps(elem, indent=4)) - return_info.append({"id": elem['id'], - "status_code": rc, - "reason": r}) + return_info.append({"id": elem["id"], "status_code": rc, "reason": r}) except TypeError as e: - return_info.append({"id": "UNK", - "status_code": status.HTTP_422_UNPROCESSABLE_ENTITY, - "reason": e.args[0]}) + return_info.append( + { + "id": "UNK", + "status_code": status.HTTP_422_UNPROCESSABLE_ENTITY, + "reason": e.args[0], + } + ) except Exception as e: raise e # reason = getattr(e, 'message', str(e)) @@ -74,7 +75,7 @@ def send_data_array(self, json_object): def send_data(self, json_object): # Send the data to a FIWARE Context Broker instance headers = { - 'Content-Type': 'application/ld+json' + "Content-Type": "application/ld+json" # , 'Accept': 'application/ld+json' } @@ -96,7 +97,7 @@ def send_data(self, json_object): if __name__ == "__main__": - c = NGSILDConnector('../common/config.json') + c = NGSILDConnector("../common/config.json") print(c.get_url()) parser = Parser() diff --git a/poetry.lock b/poetry.lock index 52df914..f13ffb7 100755 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + [[package]] name = "anyio" version = "3.6.2" @@ -5,6 +7,10 @@ description = "High level compatibility layer for multiple asynchronous event lo category = "main" optional = false python-versions = ">=3.6.2" +files = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] [package.dependencies] idna = ">=2.8" @@ -16,18 +22,16 @@ test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>= trio = ["trio (>=0.16,<0.22)"] [[package]] -name = "attrs" -version = "22.1.0" -description = "Classes Without Boilerplate" +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" category = "dev" optional = false -python-versions = ">=3.5" - -[package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] [[package]] name = "certifi" @@ -36,6 +40,22 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, +] + +[[package]] +name = "chardet" +version = "5.1.0" +description = "Universal encoding detector for Python 3" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, + {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, +] [[package]] name = "charset-normalizer" @@ -44,6 +64,10 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] [package.extras] unicode-backport = ["unicodedata2"] @@ -55,6 +79,10 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -66,6 +94,10 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "contextlib2" @@ -74,6 +106,22 @@ description = "Backports and enhancements for the contextlib module" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, + {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, +] + +[[package]] +name = "distlib" +version = "0.3.7" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, + {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, +] [[package]] name = "docopt" @@ -82,6 +130,9 @@ description = "Pythonic argument parser, that will make you smile" category = "main" optional = false python-versions = "*" +files = [ + {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, +] [[package]] name = "exceptiongroup" @@ -90,27 +141,49 @@ description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, + {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, +] [package.extras] test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.85.1" +version = "0.100.0" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "fastapi-0.100.0-py3-none-any.whl", hash = "sha256:271662daf986da8fa98dc2b7c7f61c4abdfdccfb4786d79ed8b2878f172c6d5f"}, + {file = "fastapi-0.100.0.tar.gz", hash = "sha256:acb5f941ea8215663283c10018323ba7ea737c571b67fc7e88e9469c7eb1d12e"}, +] [package.dependencies] -pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" -starlette = "0.20.4" +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<3.0.0" +starlette = ">=0.27.0,<0.28.0" +typing-extensions = ">=4.5.0" [package.extras] -all = ["email-validator (>=1.1.1,<2.0.0)", "itsdangerous (>=1.1.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "orjson (>=3.2.1,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] -dev = ["autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "pre-commit (>=2.17.0,<3.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer (>=0.4.1,<0.7.0)"] -test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.971)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "types-orjson (==3.6.2)", "types-ujson (==5.4.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] +all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "filelock" +version = "3.12.2" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, + {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, +] + +[package.extras] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "h11" @@ -119,6 +192,10 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] [[package]] name = "hi-dateinfer" @@ -127,6 +204,10 @@ description = "Infers date format from examples, by using a series of pattern ma category = "main" optional = false python-versions = "!=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "hi-dateinfer-0.4.6.tar.gz", hash = "sha256:e2ded27aeb15ee6fcda0a30c2a341acbdaaa5b95c1c4db4c4680a4dbcda35c54"}, + {file = "hi_dateinfer-0.4.6-py3-none-any.whl", hash = "sha256:46d27ccc875890315eec9d71d76c2306e84388c4a88106d2f768a921debb49e8"}, +] [package.dependencies] pytz = "*" @@ -139,6 +220,10 @@ description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] [[package]] name = "iniconfig" @@ -147,6 +232,10 @@ description = "iniconfig: brain-dead simple config-ini parsing" category = "dev" optional = false python-versions = "*" +files = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] [[package]] name = "isodate" @@ -155,56 +244,90 @@ description = "An ISO 8601 date/time/duration parser and formatter" category = "main" optional = false python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] [package.dependencies] six = "*" [[package]] name = "lark" -version = "1.1.3" +version = "1.1.7" description = "a modern parsing library" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" +files = [ + {file = "lark-1.1.7-py3-none-any.whl", hash = "sha256:9e5dc5bbf93fa1840083707285262514a0ef8a6613874af7ea1cec60468d6e92"}, + {file = "lark-1.1.7.tar.gz", hash = "sha256:be7437bf1f37ab08b355f29ff2571d77d777113d0a8c4352b0c513dced6c5a1e"}, +] [package.extras] atomic-cache = ["atomicwrites"] +interegular = ["interegular (>=0.3.1,<0.4.0)"] nearley = ["js2py"] regex = ["regex"] [[package]] name = "loguru" -version = "0.6.0" +version = "0.7.0" description = "Python logging made (stupidly) simple" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.0-py3-none-any.whl", hash = "sha256:b93aa30099fa6860d4727f1b81f8718e965bb96253fa190fab2077aaad6d15d3"}, + {file = "loguru-0.7.0.tar.gz", hash = "sha256:1612053ced6ae84d7959dd7d5e431a0532642237ec21f7fd83ac73fe539e03e1"}, +] [package.dependencies] colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] +dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v0.990)", "pre-commit (==3.2.1)", "pytest (==6.1.2)", "pytest (==7.2.1)", "pytest-cov (==2.12.1)", "pytest-cov (==4.0.0)", "pytest-mypy-plugins (==1.10.1)", "pytest-mypy-plugins (==1.9.3)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.2.0)", "tox (==3.27.1)", "tox (==4.4.6)"] [[package]] name = "packaging" -version = "21.3" +version = "23.1" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "platformdirs" +version = "3.9.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.9.1-py3-none-any.whl", hash = "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"}, + {file = "platformdirs-3.9.1.tar.gz", hash = "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421"}, +] + +[package.extras] +docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.2.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -217,6 +340,44 @@ description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, +] [package.dependencies] typing-extensions = ">=4.1.0" @@ -232,20 +393,47 @@ description = "pyparsing module - Classes and methods to define and execute pars category = "main" optional = false python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pyproject-api" +version = "1.5.3" +description = "API to interact with the python pyproject.toml based projects" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyproject_api-1.5.3-py3-none-any.whl", hash = "sha256:14cf09828670c7b08842249c1f28c8ee6581b872e893f81b62d5465bec41502f"}, + {file = "pyproject_api-1.5.3.tar.gz", hash = "sha256:ffb5b2d7cad43f5b2688ab490de7c4d3f6f15e0b819cb588c4b771567c9729eb"}, +] + +[package.dependencies] +packaging = ">=23.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "importlib-metadata (>=6.6)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "setuptools (>=67.8)", "wheel (>=0.40)"] + [[package]] name = "pytest" -version = "7.2.0" +version = "7.4.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, +] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" @@ -254,18 +442,37 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.5" +version = "0.0.6" description = "A streaming multipart parser for Python" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7" +files = [ + {file = "python_multipart-0.0.6-py3-none-any.whl", hash = "sha256:ee698bab5ef148b0a760751c261902cd096e57e10558e11aca17646b74ee1c18"}, + {file = "python_multipart-0.0.6.tar.gz", hash = "sha256:e9925a80bb668529f1b67c7fdb0a5dacdd7cbfc6fb0bff3ea443fe22bdd62132"}, +] -[package.dependencies] -six = ">=1.4.0" +[package.extras] +dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==1.7.3)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] [[package]] name = "pytz" @@ -274,41 +481,50 @@ description = "World timezone definitions, modern and historical" category = "main" optional = false python-versions = "*" +files = [ + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, +] [[package]] name = "rdflib" -version = "6.2.0" +version = "6.3.2" description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.7,<4.0" +files = [ + {file = "rdflib-6.3.2-py3-none-any.whl", hash = "sha256:36b4e74a32aa1e4fa7b8719876fb192f19ecd45ff932ea5ebbd2e417a0247e63"}, + {file = "rdflib-6.3.2.tar.gz", hash = "sha256:72af591ff704f4caacea7ecc0c5a9056b8553e0489dd4f35a9bc52dbd41522e0"}, +] [package.dependencies] -isodate = "*" -pyparsing = "*" -setuptools = "*" +isodate = ">=0.6.0,<0.7.0" +pyparsing = ">=2.1.0,<4" [package.extras] -berkeleydb = ["berkeleydb"] -dev = ["black (==22.6.0)", "flake8", "flakeheaven", "isort", "mypy", "pep8-naming", "types-setuptools"] -docs = ["myst-parser", "sphinx (<6)", "sphinx-autodoc-typehints", "sphinxcontrib-apidoc", "sphinxcontrib-kroki"] -html = ["html5lib"] -networkx = ["networkx"] -tests = ["html5lib", "pytest", "pytest-cov"] +berkeleydb = ["berkeleydb (>=18.1.0,<19.0.0)"] +html = ["html5lib (>=1.0,<2.0)"] +lxml = ["lxml (>=4.3.0,<5.0.0)"] +networkx = ["networkx (>=2.0.0,<3.0.0)"] [[package]] name = "requests" -version = "2.28.1" +version = "2.31.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" +charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -321,6 +537,10 @@ description = "Simple data validation library" category = "main" optional = false python-versions = "*" +files = [ + {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, + {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, +] [package.dependencies] contextlib2 = ">=0.5.5" @@ -332,19 +552,10 @@ description = "A lightweight package that adds security headers for Python web f category = "main" optional = false python-versions = ">=3.6" - -[[package]] -name = "setuptools" -version = "65.5.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +files = [ + {file = "secure-0.3.0-py3-none-any.whl", hash = "sha256:a93b720c7614809c131ca80e477263140107c6c212829d0a6e1f7bc8d859c608"}, + {file = "secure-0.3.0.tar.gz", hash = "sha256:6e30939d8f95bf3b8effb8a36ebb5ed57f265daeeae905e3aa9677ea538ab64e"}, +] [[package]] name = "six" @@ -353,6 +564,10 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "sniffio" @@ -361,20 +576,29 @@ description = "Sniff out which async library your code is running under" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] [[package]] name = "starlette" -version = "0.20.4" +version = "0.27.0" description = "The little ASGI library that shines." category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "starlette-0.27.0-py3-none-any.whl", hash = "sha256:918416370e846586541235ccd38a474c08b80443ed31c578a418e2209b3eef91"}, + {file = "starlette-0.27.0.tar.gz", hash = "sha256:6a6b0d042acb8d469a01eba54e9cda6cbd24ac602c4cd016723117d6a7e73b75"}, +] [package.dependencies] anyio = ">=3.4.0,<5" +typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] [[package]] name = "tomli" @@ -383,14 +607,113 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tox" +version = "4.6.4" +description = "tox is a generic virtualenv management and test command line tool" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tox-4.6.4-py3-none-any.whl", hash = "sha256:1b8f8ae08d6a5475cad9d508236c51ea060620126fd7c3c513d0f5c7f29cc776"}, + {file = "tox-4.6.4.tar.gz", hash = "sha256:5e2ad8845764706170d3dcaac171704513cc8a725655219acb62fe4380bdadda"}, +] + +[package.dependencies] +cachetools = ">=5.3.1" +chardet = ">=5.1" +colorama = ">=0.4.6" +filelock = ">=3.12.2" +packaging = ">=23.1" +platformdirs = ">=3.8" +pluggy = ">=1.2" +pyproject-api = ">=1.5.2" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} +virtualenv = ">=20.23.1" + +[package.extras] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.23.3,!=1.23.4)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.1.1)", "devpi-process (>=0.3.1)", "diff-cover (>=7.6)", "distlib (>=0.3.6)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.17.1)", "psutil (>=5.9.5)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-xdist (>=3.3.1)", "re-assert (>=1.1)", "time-machine (>=2.10)", "wheel (>=0.40)"] + +[[package]] +name = "types-docopt" +version = "0.6.11.3" +description = "Typing stubs for docopt" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-docopt-0.6.11.3.tar.gz", hash = "sha256:ee6db04587900d94f4c137f9b191cc72ca8920a2bd7f823e27b72cb8da0d9d3b"}, + {file = "types_docopt-0.6.11.3-py3-none-any.whl", hash = "sha256:6836fbc5f4f622a0a97f9dea03f62a387e03157e6559e3086f49be0ca089590a"}, +] + +[[package]] +name = "types-python-dateutil" +version = "2.8.19.14" +description = "Typing stubs for python-dateutil" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-python-dateutil-2.8.19.14.tar.gz", hash = "sha256:1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b"}, + {file = "types_python_dateutil-2.8.19.14-py3-none-any.whl", hash = "sha256:f977b8de27787639986b4e28963263fd0e5158942b3ecef91b9335c130cb1ce9"}, +] + +[[package]] +name = "types-pytz" +version = "2023.3.0.0" +description = "Typing stubs for pytz" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-pytz-2023.3.0.0.tar.gz", hash = "sha256:ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"}, + {file = "types_pytz-2023.3.0.0-py3-none-any.whl", hash = "sha256:4fc2a7fbbc315f0b6630e0b899fd6c743705abe1094d007b0e612d10da15e0f3"}, +] + +[[package]] +name = "types-requests" +version = "2.31.0.2" +description = "Typing stubs for requests" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.31.0.2.tar.gz", hash = "sha256:6aa3f7faf0ea52d728bb18c0a0d1522d9bfd8c72d26ff6f61bfc3d06a411cf40"}, + {file = "types_requests-2.31.0.2-py3-none-any.whl", hash = "sha256:56d181c85b5925cbc59f4489a57e72a8b2166f18273fd8ba7b6fe0c0b986f12a"}, +] + +[package.dependencies] +types-urllib3 = "*" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] [[package]] name = "urllib3" @@ -399,6 +722,10 @@ description = "HTTP library with thread-safe connection pooling, file post, and category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +files = [ + {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -407,18 +734,44 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "uvicorn" -version = "0.19.0" +version = "0.23.1" description = "The lightning-fast ASGI server." category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.23.1-py3-none-any.whl", hash = "sha256:1d55d46b83ee4ce82b4e82f621f2050adb3eb7b5481c13f9af1744951cae2f1f"}, + {file = "uvicorn-0.23.1.tar.gz", hash = "sha256:da9b0c8443b2d7ee9db00a345f1eee6db7317432c9d4400f5049cc8d358383be"}, +] [package.dependencies] click = ">=7.0" h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "virtualenv" +version = "20.24.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.24.1-py3-none-any.whl", hash = "sha256:01aacf8decd346cf9a865ae85c0cdc7f64c8caa07ff0d8b1dfc1733d10677442"}, + {file = "virtualenv-20.24.1.tar.gz", hash = "sha256:2ef6a237c31629da6442b0bcaa3999748108c7166318d1f55cc9f8d7294e97bd"}, +] + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.12,<4" +platformdirs = ">=3.5.1,<4" [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.0)"] +docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezer (>=0.4.6)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.8)", "time-machine (>=2.9)"] [[package]] name = "wheel" @@ -427,6 +780,10 @@ description = "A built-package format for Python" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "wheel-0.37.1-py2.py3-none-any.whl", hash = "sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a"}, + {file = "wheel-0.37.1.tar.gz", hash = "sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4"}, +] [package.extras] test = ["pytest (>=3.0.0)", "pytest-cov"] @@ -438,197 +795,15 @@ description = "A small Python utility to set file creation time on Windows" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] [package.extras] dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] -lock-version = "1.1" -python-versions = "^3.10" -content-hash = "efb208b9ebb656a11935975f1f40e583c2c4ac285bbb8a5509f560da21396f7e" - -[metadata.files] -anyio = [ - {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, - {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -certifi = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -contextlib2 = [ - {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, - {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, -] -docopt = [ - {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, - {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, -] -fastapi = [ - {file = "fastapi-0.85.1-py3-none-any.whl", hash = "sha256:de3166b6b1163dc22da4dc4ebdc3192fcbac7700dd1870a1afa44de636a636b5"}, - {file = "fastapi-0.85.1.tar.gz", hash = "sha256:1facd097189682a4ff11cbd01334a992e51b56be663b2bd50c2c09523624f144"}, -] -h11 = [ - {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, - {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, -] -hi-dateinfer = [ - {file = "hi-dateinfer-0.4.6.tar.gz", hash = "sha256:e2ded27aeb15ee6fcda0a30c2a341acbdaaa5b95c1c4db4c4680a4dbcda35c54"}, - {file = "hi_dateinfer-0.4.6-py3-none-any.whl", hash = "sha256:46d27ccc875890315eec9d71d76c2306e84388c4a88106d2f768a921debb49e8"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -isodate = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] -lark = [ - {file = "lark-1.1.3-py3-none-any.whl", hash = "sha256:45cd8b4d8b0487863f0f4cf3c305a1293db86af576d4a62cfb572e57a9b609e5"}, - {file = "lark-1.1.3.tar.gz", hash = "sha256:ca0d1aeb57f434c7276d209729e92b0e5017d177dde553134760c35bb4647d11"}, -] -loguru = [ - {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, - {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -pydantic = [ - {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, - {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, - {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, - {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, - {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, - {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, - {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, - {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, - {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, - {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -pytest = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] -python-multipart = [ - {file = "python-multipart-0.0.5.tar.gz", hash = "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43"}, -] -pytz = [ - {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, - {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, -] -rdflib = [ - {file = "rdflib-6.2.0-py3-none-any.whl", hash = "sha256:85c34a86dfc517a41e5f2425a41a0aceacc23983462b32e68610b9fad1383bca"}, - {file = "rdflib-6.2.0.tar.gz", hash = "sha256:62dc3c86d1712db0f55785baf8047f63731fa59b2682be03219cb89262065942"}, -] -requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, -] -schema = [ - {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, - {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, -] -secure = [ - {file = "secure-0.3.0-py3-none-any.whl", hash = "sha256:a93b720c7614809c131ca80e477263140107c6c212829d0a6e1f7bc8d859c608"}, - {file = "secure-0.3.0.tar.gz", hash = "sha256:6e30939d8f95bf3b8effb8a36ebb5ed57f265daeeae905e3aa9677ea538ab64e"}, -] -setuptools = [ - {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, - {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -sniffio = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] -starlette = [ - {file = "starlette-0.20.4-py3-none-any.whl", hash = "sha256:c0414d5a56297d37f3db96a84034d61ce29889b9eaccf65eb98a0b39441fcaa3"}, - {file = "starlette-0.20.4.tar.gz", hash = "sha256:42fcf3122f998fefce3e2c5ad7e5edbf0f02cf685d646a83a08d404726af5084"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] -urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, -] -uvicorn = [ - {file = "uvicorn-0.19.0-py3-none-any.whl", hash = "sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f"}, - {file = "uvicorn-0.19.0.tar.gz", hash = "sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25"}, -] -wheel = [ - {file = "wheel-0.37.1-py2.py3-none-any.whl", hash = "sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a"}, - {file = "wheel-0.37.1.tar.gz", hash = "sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4"}, -] -win32-setctime = [ - {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, - {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, -] +lock-version = "2.0" +python-versions = ">=3.8,<4.0" +content-hash = "f9a0b64ad4adc32e4fac061e17834c18b798eac5868c2a7aad1eeddd8d8c4e63" diff --git a/pyproject.toml b/pyproject.toml index 95556a6..09af394 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "SDMX2JSON-LD" -version = "0.5.2" +version = "1.0.0" description = "A SDMX in RDF Turtle 1.1 format parser to generate valid JSON-LD and send to FIWARE Context Brokers using ETSI NGSI-LD." authors = ["Fernando López "] readme = "./sdmx2jsonld/README.md" @@ -15,29 +15,33 @@ classifiers = [ repository = "https://github.com/flopezag/IoTAgent-Turtle" - [tool.poetry.dependencies] -python = "^3.7" -lark = "1.1.3" +python = ">=3.8,<4.0" +lark = "1.1.7" secure = "0.3.0" docopt = "0.6.2" schema = "0.7.5" hi-dateinfer = "0.4.6" -fastapi = "0.85.1" -uvicorn = "0.19.0" -python-multipart = "0.0.5" -loguru = "0.6.0" -requests = "2.28.1" -rdflib = "~6.2.0" +fastapi = "0.100.0" +uvicorn = "0.23.1" +python-multipart = "0.0.6" +loguru = "0.7.0" +requests = "2.31.0" +rdflib = "6.3.2" +python-dateutil = "2.8.2" [tool.poetry.dev-dependencies] -pytest = "*" +pytest = "7.4.0" +tox = "4.6.4" +types-docopt = "0.6.11.3" +types-python-dateutil = "2.8.19.14" +types-pytz = "2023.3.0.0" +types-requests = "2.31.0.2" [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" - [[tool.poetry_bumpversion.replacements]] files = ["cli/command.py", "sdmx2jsonld/transform/parser.py"] search = '__version__ = "{current_version}"' diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..65ad636 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +# python3.11, virtualenv -ppython3.11 .venv +# file with the requirements to execute the python tests using tox +pytest==7.4.0 +tox==4.6.4 +types-docopt==0.6.11.3 +types-python-dateutil==2.8.19.14 +types-pytz==2023.3.0.0 +types-requests==2.31.0.2 diff --git a/sdmx2jsonld/common/classprecedence.py b/sdmx2jsonld/common/classprecedence.py index 803af7c..8cd6936 100644 --- a/sdmx2jsonld/common/classprecedence.py +++ b/sdmx2jsonld/common/classprecedence.py @@ -22,15 +22,15 @@ entities = { - 'qb:DataStructureDefinition': 'Dataset', - 'qb:ComponentSpecification': 'Component', - 'qb:AttributeProperty': 'Attribute', - 'qb:DimensionProperty': 'Dimension', - 'qb:CodedProperty': 'Dimension', - 'rdfs:Class': 'Class', - 'owl:Class': 'Class', - 'skos:ConceptScheme': 'ConceptScheme', - 'skos:Concept': 'Range' + "qb:DataStructureDefinition": "Dataset", + "qb:ComponentSpecification": "Component", + "qb:AttributeProperty": "Attribute", + "qb:DimensionProperty": "Dimension", + "qb:CodedProperty": "Dimension", + "rdfs:Class": "Class", + "owl:Class": "Class", + "skos:ConceptScheme": "ConceptScheme", + "skos:Concept": "Range", } @@ -47,7 +47,7 @@ def __init__(self): "skos:Concept": 40, "rdfs:Class": 20, "owl:Class": 20, - "qb:SliceKey": 10 + "qb:SliceKey": 10, } def get_value(self, aclass: str) -> int: @@ -72,8 +72,7 @@ def precedence(self, data: list) -> str: raise ClassPrecedenceClassError(data) # In other case, we return the max value of the list - aux = max(classes_values) - aux = data[classes_values.index(aux)] + aux = data[classes_values.index(max(classes_values))] return aux @@ -86,11 +85,12 @@ def __init__(self, data, message): self.data = data def __str__(self): - return f'{self.data} -> {self.message}' + return f"{self.data} -> {self.message}" class ClassPrecedencePropertyError(ClassPrecedenceError): """Raised when the input value is too small""" + """Exception raised for errors in the input data. Attributes: @@ -104,6 +104,7 @@ def __init__(self, data, message="Incompatible multiclass definition"): class ClassPrecedenceClassError(ClassPrecedenceError): """Raised when the input value is too large""" + """Exception raised for errors in the input data. Attributes: @@ -115,6 +116,6 @@ def __init__(self, data, message="Possible redundant Class definition"): super().__init__(data=data, message=message) -if __name__ == '__main__': +if __name__ == "__main__": pre = Precedence() - obtained = pre.precedence(['qb:DataStructureDefinition']) + obtained = pre.precedence(["qb:DataStructureDefinition"]) diff --git a/sdmx2jsonld/common/commonclass.py b/sdmx2jsonld/common/commonclass.py index a30e31d..8392947 100644 --- a/sdmx2jsonld/common/commonclass.py +++ b/sdmx2jsonld/common/commonclass.py @@ -33,18 +33,18 @@ def __init__(self, entity): def add_context(self, context, context_mapping): # Set the context as it is received and mixed with the core context - self.data['@context'] = context['@context'] + self.data["@context"] = context["@context"] # Fix the prefix of the core properties of the Dataset entity new_data = dict() for k, v in self.data.items(): # Return if the string matched the ReGex - out = k.split(':') + out = k.split(":") if len(out) == 2 and out[0] in context_mapping.keys(): new_prefix = context_mapping[out[0]] - new_key = new_prefix + ':' + out[1] + new_key = new_prefix + ":" + out[1] new_data[new_key] = self.data[k] self.keys[k] = new_key @@ -59,15 +59,15 @@ def get(self): def save(self): data = self.get() - aux = data['id'].split(":") + aux = data["id"].split(":") length_aux = len(aux) # We need to check that the output folder exist - if exists('./output') is False: + if exists("./output") is False: # We need to create the folder because it does not exist - mkdir('./output') + mkdir("./output") - filename = './output/' + '_'.join(aux[length_aux - 2:]) + '.jsonld' + filename = "./output/" + "_".join(aux[length_aux - 2 :]) + ".jsonld" # Serializing json json_object = dumps(data, indent=4, ensure_ascii=False) @@ -86,18 +86,12 @@ def generate_id(self, value, entity=None, update_id=False): new_aux = "urn:ngsi-ld:" + entity + ":" + aux if update_id: - self.data['id'] = new_aux + self.data["id"] = new_aux return new_aux else: return aux, new_aux - def __generate_property__(self, key, value): - result = { - key: { - "type": "Property", - "value": value - } - } + result = {key: {"type": "Property", "value": value}} return result diff --git a/sdmx2jsonld/common/config.py b/sdmx2jsonld/common/config.py index 2409be9..fb80855 100644 --- a/sdmx2jsonld/common/config.py +++ b/sdmx2jsonld/common/config.py @@ -24,13 +24,15 @@ # Settings file is inside Basics directory, therefore I have to go back to the parent directory # to have the Code Home directory MODULEHOME = dirname(dirname(abspath(__file__))) -GRAMMARFOLDER = join(MODULEHOME, 'grammar') -GRAMMARFILE = join(GRAMMARFOLDER, 'grammar.lark') +GRAMMARFOLDER = join(MODULEHOME, "grammar") +GRAMMARFILE = join(GRAMMARFOLDER, "grammar.lark") if not exists(GRAMMARFILE): - msg = '\nERROR: There is not Lark grammar file in the expected folder. ' \ - '\n Unable to parse the RDF Turtle file.' \ - '\n\n Please correct it if you do not want to see these messages.\n\n\n' + msg = ( + "\nERROR: There is not Lark grammar file in the expected folder. " + "\n Unable to parse the RDF Turtle file." + "\n\n Please correct it if you do not want to see these messages.\n\n\n" + ) print(msg) diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index b444f6b..1a75e70 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -19,7 +19,7 @@ # License for the specific language governing permissions and limitations # under the License. ## -from hidateinfer import infer +from hidateinfer import infer # type: ignore[import] from datetime import datetime, timezone from re import compile, sub from dateutil import parser @@ -30,10 +30,10 @@ class DataTypeConversion: def __init__(self): self.types = { - 'xsd:dateTime': 'stoutc', - 'xsd:int': 'stoi', - 'xsd:boolean': 'stob', - 'xsd:float': 'stof' + "xsd:dateTime": "stoutc", + "xsd:int": "stoi", + "xsd:boolean": "stob", + "xsd:float": "stof", } self.regex_12hour = compile(r"(^.*T%)(I)(.*)$") @@ -57,7 +57,7 @@ def correct_datatype_format(self, format_dt: str, hour24: bool = True): def convert(self, data, datatype): def stoutc(value): """ - Converts a date in string format to UTC date using + Converts a date in string format to UTC date using """ dt = parser.parse(value, tzinfos=whois_timezone_info) dt = dt.astimezone(pytz.UTC) @@ -69,86 +69,109 @@ def stodt(value): elif isinstance(value, list): result = infer(value) else: - raise Exception(f'Invalid format received: {type(value)}') + raise Exception(f"Invalid format received: {type(value)}") result = self.correct_datatype_format(result) - result = datetime.strptime(value, result).replace(tzinfo=timezone.utc).isoformat() + result = ( + datetime.strptime(value, result) + .replace(tzinfo=timezone.utc) + .isoformat() + ) return result def stoi(value): """ - Converts 'something' to int. Raises exception for invalid formats + Converts 'something' to int. Raises exception for invalid formats """ if isinstance(value, str): - result = value.replace('"', '') + result = value.replace('"', "") elif isinstance(value, int): result = value else: - raise Exception(f'Invalid format received: {type(value)}') + raise Exception(f"Invalid format received: {type(value)}") return int(result) def stof(value): """ - Converts 'something' to float. Raises exception for invalid formats + Converts 'something' to float. Raises exception for invalid formats """ if isinstance(value, str): - result = value.replace('"', '') + result = value.replace('"', "") elif isinstance(value, float): result = value else: - raise Exception(f'Invalid format received: {type(value)}') + raise Exception(f"Invalid format received: {type(value)}") return float(result) def stob(value): """ - Converts 'something' to boolean. Raises exception for invalid formats - Possible True values: 1, True, "1", "TRue", "yes", "y", "t" - Possible False values: 0, False, None, [], {}, "", "0", "faLse", "no", "n", "f", 0.0, ... + Converts 'something' to boolean. Raises exception for invalid formats + Possible True values: 1, True, "1", "TRue", "yes", "y", "t" + Possible False values: 0, False, None, [], {}, "", "0", "faLse", "no", "n", "f", 0.0, ... """ if str(value).lower() in ("yes", "y", "true", "t", "1"): return True - if str(value).lower() in ("no", "n", "false", "f", "0", "0.0", "", "none", "[]", "{}"): + if str(value).lower() in ( + "no", + "n", + "false", + "f", + "0", + "0.0", + "", + "none", + "[]", + "{}", + ): return False # logger.error(f'Invalid value for boolean conversion: {str(value)}') - raise Exception(f'Invalid value for boolean conversion: {str(value)}') + raise Exception(f"Invalid value for boolean conversion: {str(value)}") try: # jicg - function = self.types[datatype] + f'(value="{data}")' - function = self.types[datatype] + '(value=' + data + ')' + function = self.types[datatype] + "(value=" + data + ")" return eval(function) except KeyError: # logger.error(f'Datatype not defined: {datatype}') - print(f'Datatype not defined: {datatype}') - raise Exception(f'Datatype not defined: {datatype}') + print(f"Datatype not defined: {datatype}") + raise Exception(f"Datatype not defined: {datatype}") except NameError: # logger.error(f"name '{data}' is not defined") print(f"name '{data}' is not defined") raise Exception(f"name '{data}' is not defined") -if __name__ == '__main__': +if __name__ == "__main__": from lark import Token - data1 = ['"2022-01-15T08:00:00.000"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] - data2 = ['"2"', Token('FORMATCONNECTOR', '^^'), 'xsd:int'] - data22 = ['2', Token('FORMATCONNECTOR', '^^'), 'xsd:int'] - data23 = ['asdfs', Token('FORMATCONNECTOR', '^^'), 'xsd:int'] - data3 = ['"true"', Token('FORMATCONNECTOR', '^^'), 'xsd:boolean'] - data4 = ['"fake"', Token('FORMATCONNECTOR', '^^'), 'otraCosa'] - data5 = ['"2022-01-10T09:00:00.000"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] - data6 = ['"2021-07-01T11:50:37.3"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] - data7 = ['"2021-09-28T15:31:24.05"', Token('FORMATCONNECTOR', '^^'), 'xsd:dateTime'] - - print(infer(['Mon Jan 13 09:52:52 MST 2014'])) + data1 = [ + '"2022-01-15T08:00:00.000"', + Token("FORMATCONNECTOR", "^^"), + "xsd:dateTime", + ] + data2 = ['"2"', Token("FORMATCONNECTOR", "^^"), "xsd:int"] + data22 = ["2", Token("FORMATCONNECTOR", "^^"), "xsd:int"] + data23 = ["asdfs", Token("FORMATCONNECTOR", "^^"), "xsd:int"] + data3 = ['"true"', Token("FORMATCONNECTOR", "^^"), "xsd:boolean"] + data4 = ['"fake"', Token("FORMATCONNECTOR", "^^"), "otraCosa"] + data5 = [ + '"2022-01-10T09:00:00.000"', + Token("FORMATCONNECTOR", "^^"), + "xsd:dateTime", + ] + data6 = ['"2021-07-01T11:50:37.3"', Token("FORMATCONNECTOR", "^^"), "xsd:dateTime"] + data7 = ['"2021-09-28T15:31:24.05"', Token("FORMATCONNECTOR", "^^"), "xsd:dateTime"] + + print(infer(["Mon Jan 13 09:52:52 MST 2014"])) print(infer([data1[0]])) print() - print(infer(['2022-01-15T08:00:00'])) + print(infer(["2022-01-15T08:00:00"])) print(infer([data1[0]])) print() @@ -170,7 +193,7 @@ def stob(value): try: print(dataConversionType.convert(data4[0], data4[2])) except Exception: - print('Exception') + print("Exception") # Convert datetime generated into UTC format: 2021-12-21T16:18:55Z or 2021-12-21T16:18:55+00:00, ISO8601 @@ -180,5 +203,5 @@ def stob(value): print(dataConversionType.convert(data7[0], data7[2])) - data101 = ['"3016.9"', Token('FORMATCONNECTOR', '^^'), 'xsd:float'] + data101 = ['"3016.9"', Token("FORMATCONNECTOR", "^^"), "xsd:float"] print(dataConversionType.convert(data101[0], data101[2])) diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index e80ee7e..c36dd7d 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -37,12 +37,14 @@ def filter_key_with_prefix(prefix_key, not_allowed_keys, further_process_keys): # this is a key with prefix that we want to keep return True else: - if aux[1] not in ['component', 'label']: + if aux[1] not in ["component", "label"]: # These are the identified not allowed keys, we need to inform about them - logger.warning(f'The property {aux[1]} is not supported in statDCAT-AP') + logger.warning(f"The property {aux[1]} is not supported in statDCAT-AP") else: # These are the identified keys managed in a different way - logger.info(f'The property {aux[1]} is manage afterwards in Dataset Class or in Property Class') + logger.info( + f"The property {aux[1]} is manage afterwards in Dataset Class or in Property Class" + ) return False else: @@ -70,9 +72,9 @@ def flatten_value(y): result.append(flatten_value(y[i])) return result else: # in case of len == 0 be return the empty string - return '' + return "" else: - return y.replace('"', '') + return y.replace('"', "") def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): @@ -84,7 +86,11 @@ def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): # We need to get the list of keys from the dict new_keys = list( - filter(lambda x: filter_key_with_prefix(x, not_allowed_keys, further_process_keys), list(aux.keys()))) + filter( + lambda x: filter_key_with_prefix(x, not_allowed_keys, further_process_keys), + list(aux.keys()), + ) + ) new_data = {k: aux[k] for k in new_keys} corrected_dict = {k.replace(k, extract_prefix(k)): v for k, v in new_data.items()} @@ -95,16 +101,20 @@ def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): def extract_prefix(attribute): result = None if attribute is None or len(attribute) == 0: - raise ClassExtractPrefixError(data=attribute, message=f"Unexpected data received: '{attribute}'") + raise ClassExtractPrefixError( + data=attribute, message=f"Unexpected data received: '{attribute}'" + ) else: - data = attribute.split(':') + data = attribute.split(":") if len(data) == 1: result = data[0] elif len(data) == 2: result = data[1] else: - raise ClassExtractPrefixError(data=attribute, message=f"Unexpected number of prefixes: '{attribute}'") + raise ClassExtractPrefixError( + data=attribute, message=f"Unexpected number of prefixes: '{attribute}'" + ) return result @@ -112,7 +122,7 @@ def extract_prefix(attribute): def get_property_value(data, property_name): # At the moment, we only find the first occurs of the property i = 0 - key = '' + key = "" found = -1 for i in range(0, len(data)): key = data[i] @@ -123,6 +133,6 @@ def get_property_value(data, property_name): break if found != -1: - return i, key, data[i+1] + return i, key, data[i + 1] else: - return -1, '', '' + return -1, "", "" diff --git a/sdmx2jsonld/common/regparser.py b/sdmx2jsonld/common/regparser.py index 1a2570d..bcdfdb6 100644 --- a/sdmx2jsonld/common/regparser.py +++ b/sdmx2jsonld/common/regparser.py @@ -30,14 +30,14 @@ def __init__(self): # Compile the Regex self.re = re.compile(regex) - def obtain_id(self, string_to_parse, prefix_string=''): + def obtain_id(self, string_to_parse, prefix_string=""): # Return if the string matched the ReGex out = self.re.match(string_to_parse) if out is None: # Check if the prefixed name include ':' try: - obtained_id = string_to_parse.split(':')[1] + obtained_id = string_to_parse.split(":")[1] except IndexError: # We have a normal prefix or data obtained_id = string_to_parse @@ -47,6 +47,6 @@ def obtain_id(self, string_to_parse, prefix_string=''): out = out.split("/") # we get the last value which corresponds to the id - obtained_id = prefix_string + out[(len(out) - 1):][0] + obtained_id = prefix_string + out[(len(out) - 1) :][0] return obtained_id diff --git a/sdmx2jsonld/cube/measuretype.py b/sdmx2jsonld/cube/measuretype.py index 610691a..3aee669 100644 --- a/sdmx2jsonld/cube/measuretype.py +++ b/sdmx2jsonld/cube/measuretype.py @@ -31,11 +31,13 @@ def __init__(self): # rdfs:comment "Generic measure dimension, the value of this dimension indicates which measure (from the set of measures in the DSD) is being given by the obsValue (or other primary measure)"@en; # rdfs:range qb:MeasureProperty; # rdfs:isDefinedBy ; - super().__init__(entity_id='measureType', - label='measure type', - description='Generic measure dimension, the value of this dimension indicates which measure ' - '(from the set of measures in the DSD) is being given by the obsValue ' - '(or other primary measure).', - concept_id=None, - identifier='measureType', - entity_range='xsd:string') + super().__init__( + entity_id="measureType", + label="measure type", + description="Generic measure dimension, the value of this dimension indicates which measure " + "(from the set of measures in the DSD) is being given by the obsValue " + "(or other primary measure).", + concept_id=None, + identifier="measureType", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/exceptions/__init__.py b/sdmx2jsonld/exceptions/__init__.py index 575252e..dd06aa0 100644 --- a/sdmx2jsonld/exceptions/__init__.py +++ b/sdmx2jsonld/exceptions/__init__.py @@ -26,9 +26,9 @@ class UnexpectedEOF(LarkUnexpectedEOF): def __init__(self, expected, state=None, terminals_by_name=None): - super(LarkUnexpectedEOF, self).__init__(expected=expected, - state=state, - terminals_by_name=terminals_by_name) + super(LarkUnexpectedEOF, self).__init__( + expected=expected, state=state, terminals_by_name=terminals_by_name + ) class UnexpectedInput(LarkUnexpectedInput): @@ -37,11 +37,22 @@ def __init__(self): class UnexpectedToken(LarkUnexpectedToken): - def __init__(self, token, expected, considered_rules=None, state=None, interactive_parser=None, terminals_by_name=None, token_history=None): - super(LarkUnexpectedToken, self).__init__(token=token, - expected=expected, - considered_rules=considered_rules, - state=state, - interactive_parser=interactive_parser, - terminals_by_name=terminals_by_name, - token_history=token_history) + def __init__( + self, + token, + expected, + considered_rules=None, + state=None, + interactive_parser=None, + terminals_by_name=None, + token_history=None, + ): + super(LarkUnexpectedToken, self).__init__( + token=token, + expected=expected, + considered_rules=considered_rules, + state=state, + interactive_parser=interactive_parser, + terminals_by_name=terminals_by_name, + token_history=token_history, + ) diff --git a/sdmx2jsonld/exceptions/exceptions.py b/sdmx2jsonld/exceptions/exceptions.py index 83d7416..509d3ca 100644 --- a/sdmx2jsonld/exceptions/exceptions.py +++ b/sdmx2jsonld/exceptions/exceptions.py @@ -27,11 +27,12 @@ def __init__(self, data, message): self.data = data def __str__(self): - return f'{self.data} -> {self.message}' + return f"{self.data} -> {self.message}" class ClassConfStatusError(ClassSDMXAttributeError): """Raised when the input value is not included in the list of available values for confStatus""" + """Exception raised for errors in the input data. Attributes: @@ -45,6 +46,7 @@ def __init__(self, data, message="ConfStatus value is not the expected"): class ClassObsStatusError(ClassSDMXAttributeError): """Raised when the input value is not included in the list of available values for obsStatus""" + """Exception raised for errors in the input data. Attributes: @@ -58,6 +60,7 @@ def __init__(self, data, message="ObsStatus value is not the expected"): class ClassCode(ClassSDMXAttributeError): """Raised when the input value is not included in the list of available values for unitMult and decimals""" + """Exception raised for errors in the input data. Attributes: @@ -71,6 +74,7 @@ def __init__(self, data, message="Decimals value is not the expected"): class ClassFreqError(ClassSDMXAttributeError): """Raised when the input value is not included in the list of available values for Freq""" + """Exception raised for errors in the input data. Attributes: @@ -84,6 +88,7 @@ def __init__(self, data, message="Decimals value is not the expected"): class ClassExtractPrefixError(ClassSDMXAttributeError): """Raised when the input value is None or Empty or includes several prefixes""" + """Exception raised for errors in the input data. Attributes: diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 506d2d9..3e55133 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -43,14 +43,14 @@ def fix_value(self, value): # any other value will return an error number: int() = 0 - m = search(f'sdmx-code:{self.typecode}-(.*)', str(value)) + m = search(f"sdmx-code:{self.typecode}-(.*)", str(value)) if m is not None: number = int(m.group(1)) else: # The data is not following the sdmx-code: we have to check which one # 1) Check if there is a value without the prefix - m = search(f'{self.typecode}-(.*)', str(value)) + m = search(f"{self.typecode}-(.*)", str(value)) if m is not None: number = int(m.group(1)) @@ -63,11 +63,14 @@ def fix_value(self, value): try: number = int(value) except ValueError: - raise ClassCode(data=value, - message=f'Data is not a valid value') + raise ClassCode( + data=value, message=f"Data is not a valid value" + ) if number not in self.data_range: - raise ClassCode(data=value, - message=f'{self.typecode} out of range, got: {number} {self.data_range}') + raise ClassCode( + data=value, + message=f"{self.typecode} out of range, got: {number} {self.data_range}", + ) return number diff --git a/sdmx2jsonld/sdmxattributes/compilingorg.py b/sdmx2jsonld/sdmxattributes/compilingorg.py index 8719865..45434e0 100644 --- a/sdmx2jsonld/sdmxattributes/compilingorg.py +++ b/sdmx2jsonld/sdmxattributes/compilingorg.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Compiling agency"@en ; # rdfs:comment """The organisation compiling the data being reported."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='compilingOrg', - label='Compiling agency', - description='The organisation compiling the data being reported.', - concept_id='compilingOrg', - identifier='compilingOrg', - entity_range='xsd:string') + super().__init__( + entity_id="compilingOrg", + label="Compiling agency", + description="The organisation compiling the data being reported.", + concept_id="compilingOrg", + identifier="compilingOrg", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/confirmationStatus.py b/sdmx2jsonld/sdmxattributes/confirmationStatus.py index dcf583f..1b60374 100644 --- a/sdmx2jsonld/sdmxattributes/confirmationStatus.py +++ b/sdmx2jsonld/sdmxattributes/confirmationStatus.py @@ -25,20 +25,7 @@ class ConfStatus(SDMXAttribute): - status: list = [ - "F", - "N", - "C", - "D", - "S", - "A", - "O", - "T", - "G", - "M", - "E", - "P" - ] + status: list = ["F", "N", "C", "D", "S", "A", "O", "T", "G", "M", "E", "P"] def __init__(self): # sdmx-attribute:confStatus a qb:AttributeProperty, rdf:Property ; @@ -47,13 +34,15 @@ def __init__(self): # rdfs:comment """Information about the confidentiality status of the object to which this # attribute is attached."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='confStatus', - label='Confidentiality - status', - description='Information about the confidentiality status of the object ' - 'to which this attribute is attached.', - concept_id='confStatus', - identifier='confStatus', - entity_range='xsd:string') + super().__init__( + entity_id="confStatus", + label="Confidentiality - status", + description="Information about the confidentiality status of the object " + "to which this attribute is attached.", + concept_id="confStatus", + identifier="confStatus", + entity_range="xsd:string", + ) def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value @@ -66,7 +55,7 @@ def fix_value(self, value): return value_upper else: # we could receive a value in the format confStatus- - m = search('CONFSTATUS-(.*)', value_upper) + m = search("CONFSTATUS-(.*)", value_upper) if m is not None: status = m.group(1) @@ -74,9 +63,11 @@ def fix_value(self, value): if status in self.status: return status else: - message = f"ConfStatus value is not included in the list of available values,\n" \ - f" got:{value}\n" \ - f" expected:{['confStatus-'+x for x in self.status]}" + message = ( + f"ConfStatus value is not included in the list of available values,\n" + f" got:{value}\n" + f" expected:{['confStatus-'+x for x in self.status]}" + ) raise ClassConfStatusError(data=value, message=message) diff --git a/sdmx2jsonld/sdmxattributes/currency.py b/sdmx2jsonld/sdmxattributes/currency.py index 73b65bf..a657e6e 100644 --- a/sdmx2jsonld/sdmxattributes/currency.py +++ b/sdmx2jsonld/sdmxattributes/currency.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Currency"@en ; # rdfs:comment """Monetary denomination of the object being measured."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='currency', - label='Currency', - description='Monetary denomination of the object being measured.', - concept_id='currency', - identifier='currency', - entity_range='xsd:string') + super().__init__( + entity_id="currency", + label="Currency", + description="Monetary denomination of the object being measured.", + concept_id="currency", + identifier="currency", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/dataComp.py b/sdmx2jsonld/sdmxattributes/dataComp.py index e5cf8b3..58c3fe4 100644 --- a/sdmx2jsonld/sdmxattributes/dataComp.py +++ b/sdmx2jsonld/sdmxattributes/dataComp.py @@ -30,10 +30,12 @@ def __init__(self): # rdfs:comment """Operations performed on data to derive new information according # to a given set of rules."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='dataComp', - label='Data Compilation', - description='Operations performed on data to derive new information according to a ' - 'given set of rules.', - concept_id='dataComp', - identifier='dataComp', - entity_range='xsd:string') + super().__init__( + entity_id="dataComp", + label="Data Compilation", + description="Operations performed on data to derive new information according to a " + "given set of rules.", + concept_id="dataComp", + identifier="dataComp", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/decimals.py b/sdmx2jsonld/sdmxattributes/decimals.py index a1828a5..716f4bf 100644 --- a/sdmx2jsonld/sdmxattributes/decimals.py +++ b/sdmx2jsonld/sdmxattributes/decimals.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Decimals"@en ; # rdfs:comment """The number of digits of an observation to the right of a decimal point."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='decimals', - label='Decimals', - description='The number of digits of an observation to the right of a decimal point.', - concept_id='decimals', - identifier='decimals', - entity_range='xsd:integer') + super().__init__( + entity_id="decimals", + label="Decimals", + description="The number of digits of an observation to the right of a decimal point.", + concept_id="decimals", + identifier="decimals", + entity_range="xsd:integer", + ) diff --git a/sdmx2jsonld/sdmxattributes/dissorg.py b/sdmx2jsonld/sdmxattributes/dissorg.py index d7ea500..01b57eb 100644 --- a/sdmx2jsonld/sdmxattributes/dissorg.py +++ b/sdmx2jsonld/sdmxattributes/dissorg.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Data Dissemination Agency"@en ; # rdfs:comment """The organisation disseminating the data."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='dissOrg', - label='Data Dissemination Agency', - description='The organisation disseminating the data.', - concept_id='dissOrg', - identifier='dissOrg', - entity_range='xsd:string') + super().__init__( + entity_id="dissOrg", + label="Data Dissemination Agency", + description="The organisation disseminating the data.", + concept_id="dissOrg", + identifier="dissOrg", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/observationStatus.py b/sdmx2jsonld/sdmxattributes/observationStatus.py index 2410a54..e5b7f9a 100644 --- a/sdmx2jsonld/sdmxattributes/observationStatus.py +++ b/sdmx2jsonld/sdmxattributes/observationStatus.py @@ -45,7 +45,7 @@ class ObsStatus(SDMXAttribute): "J", "N", "U", - "V" + "V", ] def __init__(self): @@ -54,12 +54,14 @@ def __init__(self): # rdfs:label "Observation Status"@en ; # rdfs:comment """Information on the quality of a value or an unusual or missing value."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='obsStatus', - label='Observation Status', - description='Information on the quality of a value or an unusual or missing value.', - concept_id='obsStatus', - identifier='obsStatus', - entity_range='xsd:string') + super().__init__( + entity_id="obsStatus", + label="Observation Status", + description="Information on the quality of a value or an unusual or missing value.", + concept_id="obsStatus", + identifier="obsStatus", + entity_range="xsd:string", + ) def fix_value(self, value): # Need to check if the value received is in the list of possible values -> return that value @@ -72,7 +74,7 @@ def fix_value(self, value): return value_upper else: # we could receive a value in the format obsStatus- - m = search('OBSSTATUS-(.*)', value_upper) + m = search("OBSSTATUS-(.*)", value_upper) if m is not None: status = m.group(1) @@ -80,9 +82,11 @@ def fix_value(self, value): if status in self.status: return status else: - message = f"ObsStatus value is not included in the list of available values,\n" \ - f" got:{value}\n" \ - f" expected:{['obsStatus-'+x for x in self.status]}" + message = ( + f"ObsStatus value is not included in the list of available values,\n" + f" got:{value}\n" + f" expected:{['obsStatus-'+x for x in self.status]}" + ) raise ClassObsStatusError(data=value, message=message) diff --git a/sdmx2jsonld/sdmxattributes/sdmxattribute.py b/sdmx2jsonld/sdmxattributes/sdmxattribute.py index 2f587fa..4a33f8b 100644 --- a/sdmx2jsonld/sdmxattributes/sdmxattribute.py +++ b/sdmx2jsonld/sdmxattributes/sdmxattribute.py @@ -23,40 +23,33 @@ class SDMXAttribute(CommonClass): - def __init__(self, entity_id, label, description, concept_id, identifier, entity_range): - super().__init__(entity='AttributeProperty') + def __init__( + self, entity_id, label, description, concept_id, identifier, entity_range + ): + super().__init__(entity="AttributeProperty") self.data = { "id": f"urn:ngsi-ld:AttributeProperty:{entity_id}", "type": "AttributeProperty", - "language": { - "type": "Property", - "value": ["en"] - }, + "language": {"type": "Property", "value": ["en"]}, "label": { "type": "Property", "value": { "en": label, - } + }, }, "description": { "type": "Property", "value": { "en": description, - } + }, }, "concept": { "type": "Relationship", - "object": f"urn:ngsi-ld:Concept:{concept_id}" - }, - "identifier": { - "type": "Property", - "value": identifier - }, - "range": { - "type": "Property", - "value": entity_range + "object": f"urn:ngsi-ld:Concept:{concept_id}", }, + "identifier": {"type": "Property", "value": identifier}, + "range": {"type": "Property", "value": entity_range}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } diff --git a/sdmx2jsonld/sdmxattributes/timeFormat.py b/sdmx2jsonld/sdmxattributes/timeFormat.py index 9813a9e..3db8273 100644 --- a/sdmx2jsonld/sdmxattributes/timeFormat.py +++ b/sdmx2jsonld/sdmxattributes/timeFormat.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Time Format"@en ; # rdfs:comment """Technical format in which time is represented for the measured phenomenon."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='timeFormat', - label='Time Format', - description='Technical format in which time is represented for the measured phenomenon.', - concept_id='timeFormat', - identifier='timeFormat', - entity_range='xsd:string') + super().__init__( + entity_id="timeFormat", + label="Time Format", + description="Technical format in which time is represented for the measured phenomenon.", + concept_id="timeFormat", + identifier="timeFormat", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/timePerCollect.py b/sdmx2jsonld/sdmxattributes/timePerCollect.py index 80cefc0..f219ec0 100644 --- a/sdmx2jsonld/sdmxattributes/timePerCollect.py +++ b/sdmx2jsonld/sdmxattributes/timePerCollect.py @@ -31,11 +31,13 @@ def __init__(self): # (such as middle, average or end of period) to compile the indicator # for the target reference period."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='timePerCollect', - label='Time Period - collection', - description='Dates or periods during which the observations have been collected ' - '(such as middle, average or end of period) to compile the indicator ' - 'for the target reference period.', - concept_id='timePerCollect', - identifier='timePerCollect', - entity_range='xsd:string') + super().__init__( + entity_id="timePerCollect", + label="Time Period - collection", + description="Dates or periods during which the observations have been collected " + "(such as middle, average or end of period) to compile the indicator " + "for the target reference period.", + concept_id="timePerCollect", + identifier="timePerCollect", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/title.py b/sdmx2jsonld/sdmxattributes/title.py index bcbb5a4..e8e257a 100644 --- a/sdmx2jsonld/sdmxattributes/title.py +++ b/sdmx2jsonld/sdmxattributes/title.py @@ -29,9 +29,11 @@ def __init__(self): # rdfs:label "Title"@en ; # rdfs:comment """Textual label used as identification of a statistical object."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='title', - label='Title', - description='Textual label used as identification of a statistical object.', - concept_id='title', - identifier='title', - entity_range='xsd:string') + super().__init__( + entity_id="title", + label="Title", + description="Textual label used as identification of a statistical object.", + concept_id="title", + identifier="title", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxattributes/unitmult.py b/sdmx2jsonld/sdmxattributes/unitmult.py index 913baa5..6ddc910 100644 --- a/sdmx2jsonld/sdmxattributes/unitmult.py +++ b/sdmx2jsonld/sdmxattributes/unitmult.py @@ -30,10 +30,12 @@ def __init__(self): # rdfs:comment """Exponent in base 10 specified so that multiplying the observation # numeric values by 10^UNIT_MULT gives a value expressed in the UNIT."""@en ; # rdfs:isDefinedBy . - super().__init__(entity_id='unitMult', - label='Unit Multiplier', - description='Exponent in base 10 specified so that multiplying the observation numeric ' - 'values by 10^UNIT_MULT gives a value expressed in the UNIT.', - concept_id='unitMult', - identifier='unitMult', - entity_range='xsd:integer') + super().__init__( + entity_id="unitMult", + label="Unit Multiplier", + description="Exponent in base 10 specified so that multiplying the observation numeric " + "values by 10^UNIT_MULT gives a value expressed in the UNIT.", + concept_id="unitMult", + identifier="unitMult", + entity_range="xsd:integer", + ) diff --git a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py index a7f2dc1..96d8e94 100644 --- a/sdmx2jsonld/sdmxconcepts/cogconceptschema.py +++ b/sdmx2jsonld/sdmxconcepts/cogconceptschema.py @@ -27,22 +27,17 @@ def __init__(self): # sdmx-concept:cog a skos:ConceptScheme; # rdfs:label "Content Oriented Guidelines concept scheme"@en; # rdfs:isDefinedBy . - super().__init__(entity='ConceptSchema') + super().__init__(entity="ConceptSchema") self.data = { "id": "urn:ngsi-ld:ConceptSchema:cog", "type": "ConceptScheme", - "language": { - "type": "Property", - "value": ["en"] - }, + "language": {"type": "Property", "value": ["en"]}, "prefLabel": { "type": "Property", - "value": { - "en": "Content Oriented Guidelines concept scheme" - } + "value": {"en": "Content Oriented Guidelines concept scheme"}, }, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } diff --git a/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py b/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py index 3f1168d..90275bb 100644 --- a/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py +++ b/sdmx2jsonld/sdmxconcepts/compilingorgconcept.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].COMPILING_ORG"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='compilingOrg', - label='Compiling agency', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=' - 'SDMX:CROSS_DOMAIN_CONCEPTS[1.0].COMPILING_ORG') + super().__init__( + entity_id="compilingOrg", + label="Compiling agency", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=" + "SDMX:CROSS_DOMAIN_CONCEPTS[1.0].COMPILING_ORG", + ) diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py index df7250e..f9e1f5a 100644 --- a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -33,7 +33,9 @@ def __init__(self): # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS"; # skos:broader sdmx-concept:conf; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='confStatus', - label='Confidentiality - status', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS') + super().__init__( + entity_id="confStatus", + label="Confidentiality - status", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS", + ) diff --git a/sdmx2jsonld/sdmxconcepts/currencyconcept.py b/sdmx2jsonld/sdmxconcepts/currencyconcept.py index 3cba4f7..d4543ab 100644 --- a/sdmx2jsonld/sdmxconcepts/currencyconcept.py +++ b/sdmx2jsonld/sdmxconcepts/currencyconcept.py @@ -31,7 +31,8 @@ def __init__(self): # skos:notation # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CURRENCY"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='currency', - label='Currency', - notation= - 'urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CURRENCY') + super().__init__( + entity_id="currency", + label="Currency", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].CURRENCY", + ) diff --git a/sdmx2jsonld/sdmxconcepts/datacompconcept.py b/sdmx2jsonld/sdmxconcepts/datacompconcept.py index 02c189e..6b97de9 100644 --- a/sdmx2jsonld/sdmxconcepts/datacompconcept.py +++ b/sdmx2jsonld/sdmxconcepts/datacompconcept.py @@ -32,7 +32,9 @@ def __init__(self): # skos:notation # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='dataComp', - label='Data Compilation', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=' - 'SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP') + super().__init__( + entity_id="dataComp", + label="Data Compilation", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=" + "SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP", + ) diff --git a/sdmx2jsonld/sdmxconcepts/decimals.py b/sdmx2jsonld/sdmxconcepts/decimals.py index c357189..6ce8866 100644 --- a/sdmx2jsonld/sdmxconcepts/decimals.py +++ b/sdmx2jsonld/sdmxconcepts/decimals.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: # CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='decimals', - label='Decimals', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS') + super().__init__( + entity_id="decimals", + label="Decimals", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS", + ) diff --git a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py index a81659a..e3a4b8f 100644 --- a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py +++ b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: # CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='dissOrg', - label='Data Dissemination Agency', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG') + super().__init__( + entity_id="dissOrg", + label="Data Dissemination Agency", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG", + ) diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index ba97a74..8aa2a0b 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -30,7 +30,9 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].FREQ"; # skos:inScheme sdmx-concept:cog. - super().__init__(entity_id='freq', - label='Frequency', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].FREQ') + super().__init__( + entity_id="freq", + label="Frequency", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].FREQ", + ) diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py index f3acf69..6a6aeb4 100644 --- a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: # CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='obsStatus', - label='Observation Status', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS') + super().__init__( + entity_id="obsStatus", + label="Observation Status", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS", + ) diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index 22afa08..232b163 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -30,7 +30,9 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='refArea', - label='Reference Area', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA') + super().__init__( + entity_id="refArea", + label="Reference Area", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA", + ) diff --git a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py index 3163659..fdbbf16 100644 --- a/sdmx2jsonld/sdmxconcepts/sdmxconcept.py +++ b/sdmx2jsonld/sdmxconcepts/sdmxconcept.py @@ -24,29 +24,18 @@ class SDMXConcept(CommonClass): def __init__(self, entity_id, label, notation): - super().__init__(entity='Concept') + super().__init__(entity="Concept") self.data = { "id": f"urn:ngsi-ld:Concept:{entity_id}", "type": "Concept", - "language": { - "type": "Property", - "value": ["en"] - }, + "language": {"type": "Property", "value": ["en"]}, "inScheme": { "type": "Relationship", - "object": "urn:ngsi-ld:ConceptSchema:cog" - }, - "prefLabel": { - "type": "Property", - "value": { - "en": label - } - }, - "notation": { - "type": "Property", - "value": notation + "object": "urn:ngsi-ld:ConceptSchema:cog", }, + "prefLabel": {"type": "Property", "value": {"en": label}}, + "notation": {"type": "Property", "value": notation}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } diff --git a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py index 3a65d8f..20a6867 100644 --- a/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py +++ b/sdmx2jsonld/sdmxconcepts/timePerCollectConcept.py @@ -34,7 +34,9 @@ def __init__(self): # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT"; # skos:broader sdmx-concept:timePeriod; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='timePerCollect', - label='Time Period - collection', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT') + super().__init__( + entity_id="timePerCollect", + label="Time Period - collection", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].TIME_PER_COLLECT", + ) diff --git a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py index fc2ab45..85c7e30 100644 --- a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: # CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='timeFormat', - label='Time Format', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT') + super().__init__( + entity_id="timeFormat", + label="Time Format", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT", + ) diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index 50c83ea..6e5acc0 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -31,7 +31,9 @@ def __init__(self): # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX: # CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD"; # skos:inScheme sdmx-concept:cog. - super().__init__(entity_id='timePeriod', - label='Time Period', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD') + super().__init__( + entity_id="timePeriod", + label="Time Period", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD", + ) diff --git a/sdmx2jsonld/sdmxconcepts/titleConcept.py b/sdmx2jsonld/sdmxconcepts/titleConcept.py index bf65f68..539e99e 100644 --- a/sdmx2jsonld/sdmxconcepts/titleConcept.py +++ b/sdmx2jsonld/sdmxconcepts/titleConcept.py @@ -30,7 +30,9 @@ def __init__(self): # rdfs:isDefinedBy ; # skos:notation "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].TITLE"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='title', - label='Title', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].TITLE') + super().__init__( + entity_id="title", + label="Title", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].TITLE", + ) diff --git a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py index de24d9b..b01f8d4 100644 --- a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py +++ b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py @@ -32,7 +32,9 @@ def __init__(self): # skos:notation # "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT"; # skos:inScheme sdmx-concept:cog . - super().__init__(entity_id='unitMult', - label='Unit Multiplier', - notation='urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:' - 'CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT') + super().__init__( + entity_id="unitMult", + label="Unit Multiplier", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" + "CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT", + ) diff --git a/sdmx2jsonld/sdmxdimensions/frequency.py b/sdmx2jsonld/sdmxdimensions/frequency.py index 6446d2f..4e44f6c 100644 --- a/sdmx2jsonld/sdmxdimensions/frequency.py +++ b/sdmx2jsonld/sdmxdimensions/frequency.py @@ -32,12 +32,14 @@ def __init__(self): # rdfs:label "Frequency"@en; # rdfs:comment """The time interval at which observations occur over a given time period."""@en; # rdfs:isDefinedBy . - super().__init__(entity_id='freq', - label='Frequency', - description='The time interval at which observations occur over a given time period.', - concept_id='freq', - identifier='freq', - entity_range='xsd:string') + super().__init__( + entity_id="freq", + label="Frequency", + description="The time interval at which observations occur over a given time period.", + concept_id="freq", + identifier="freq", + entity_range="xsd:string", + ) @staticmethod def fix_value(value): @@ -47,7 +49,7 @@ def fix_value(value): # any other value will return an error value_upper = value.upper() - m = search('FREQ-(.*)', value_upper) + m = search("FREQ-(.*)", value_upper) if m is not None: status = m.group(1) diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index 8e70680..f907570 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -30,10 +30,12 @@ def __init__(self): # rdfs:label "Reference Area"@en; # rdfs:comment "The country or geographic area to which the measured statistical phenomenon relates."@en; # rdfs:isDefinedBy . - super().__init__(entity_id='refArea', - label='Reference Area', - description='The country or geographic area to which the measured statistical ' - 'phenomenon relates.', - concept_id='refArea', - identifier='refArea', - entity_range='xsd:string') + super().__init__( + entity_id="refArea", + label="Reference Area", + description="The country or geographic area to which the measured statistical " + "phenomenon relates.", + concept_id="refArea", + identifier="refArea", + entity_range="xsd:string", + ) diff --git a/sdmx2jsonld/sdmxdimensions/sdmxdimension.py b/sdmx2jsonld/sdmxdimensions/sdmxdimension.py index 8a59390..dd615c0 100644 --- a/sdmx2jsonld/sdmxdimensions/sdmxdimension.py +++ b/sdmx2jsonld/sdmxdimensions/sdmxdimension.py @@ -23,50 +23,49 @@ class SDMXDimension(CommonClass): - def __init__(self, entity_id, identifier, entity_range, label=None, description=None, concept_id=None): - super().__init__(entity='DimensionProperty') + def __init__( + self, + entity_id, + identifier, + entity_range, + label=None, + description=None, + concept_id=None, + ): + super().__init__(entity="DimensionProperty") self.data = { "id": f"urn:ngsi-ld:DimensionProperty:{entity_id}", "type": "DimensionProperty", - "language": { - "type": "Property", - "value": ["en"] - }, + "language": {"type": "Property", "value": ["en"]}, "label": { "type": "Property", "value": { "en": label, - } + }, }, "description": { "type": "Property", "value": { "en": description, - } + }, }, "concept": { "type": "Relationship", - "object": f"urn:ngsi-ld:Concept:{concept_id}" - }, - "identifier": { - "type": "Property", - "value": identifier - }, - "range": { - "type": "Property", - "value": entity_range + "object": f"urn:ngsi-ld:Concept:{concept_id}", }, + "identifier": {"type": "Property", "value": identifier}, + "range": {"type": "Property", "value": entity_range}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } # We need to check if some of the parameters are None, in that case we have to pop the key from data if label is None: - self.data.pop('label') + self.data.pop("label") if description is None: - self.data.pop('description') + self.data.pop("description") if concept_id is None: - self.data.pop('concept') + self.data.pop("concept") diff --git a/sdmx2jsonld/sdmxdimensions/timeperiod.py b/sdmx2jsonld/sdmxdimensions/timeperiod.py index 24e4479..1f112d5 100644 --- a/sdmx2jsonld/sdmxdimensions/timeperiod.py +++ b/sdmx2jsonld/sdmxdimensions/timeperiod.py @@ -32,12 +32,14 @@ def __init__(self): # rdfs:label "Time Period"@en; # rdfs:comment "The period of time or point in time to which the measured observation refers."@en; # rdfs:isDefinedBy . - super().__init__(entity_id='timePeriod', - label='Time Period', - description='The period of time or point in time to which the measured observation refers.', - concept_id='timePeriod', - identifier='timePeriod', - entity_range='xsd:string') + super().__init__( + entity_id="timePeriod", + label="Time Period", + description="The period of time or point in time to which the measured observation refers.", + concept_id="timePeriod", + identifier="timePeriod", + entity_range="xsd:string", + ) @staticmethod def fix_value(value): @@ -47,7 +49,7 @@ def fix_value(value): # any other value will return an error value_upper = value.upper() - m = search('FREQ-(.*)', value_upper) + m = search("FREQ-(.*)", value_upper) if m is not None: status = m.group(1) diff --git a/sdmx2jsonld/transform/attribute.py b/sdmx2jsonld/transform/attribute.py index 612137e..f2b3d06 100644 --- a/sdmx2jsonld/transform/attribute.py +++ b/sdmx2jsonld/transform/attribute.py @@ -25,11 +25,11 @@ class Attribute(Property): def __init__(self): - super().__init__(entity='AttributeProperty') - self.data['type'] = 'AttributeProperty' + super().__init__(entity="AttributeProperty") + self.data["type"] = "AttributeProperty" def add_data(self, attribute_id, data): super().add_data(property_id=attribute_id, data=data) # Add the id - self.data['id'] = "urn:ngsi-ld:AttributeProperty:" + attribute_id + self.data["id"] = "urn:ngsi-ld:AttributeProperty:" + attribute_id diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7d77350..7e86123 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -31,23 +31,13 @@ class Catalogue(CommonClass): def __init__(self): - super().__init__(entity='Catalogue') + super().__init__(entity="Catalogue") self.data = { "id": str(), "type": "Catalogue", - - "dataset": { - "type": "Relationship", - "object": str() - }, - - "language": { - "type": "Property", - "value": list() - }, - - + "dataset": {"type": "Relationship", "object": str()}, + "language": {"type": "Property", "value": list()}, ################################################# # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker @@ -57,26 +47,12 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "description": { - "type": "Property", - "value": dict() - }, - - "publisher": { - "type": "Property", - "value": str() - }, - - - "title": { - "type": "Property", - "value": list() - }, - + "description": {"type": "Property", "value": dict()}, + "publisher": {"type": "Property", "value": str()}, + "title": {"type": "Property", "value": list()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] - + ], } self.concept_id = str() @@ -90,36 +66,41 @@ def add_dataset(self, dataset_id): hash1 = "%032x" % random_bits # Add the id - self.data['id'] = "urn:ngsi-ld:Catalogue:" + hash1 + self.data["id"] = "urn:ngsi-ld:Catalogue:" + hash1 # Add dataset id - self.data['dataset']['object'] = dataset_id + self.data["dataset"]["object"] = dataset_id def add_data(self, title, dataset_id, data): # We need to complete the data corresponding to the Catalogue: rdfs:label self.__complete_label__(title=title, data=data) # Add the title - key = self.keys['title'] - self.data[key]['value'] = title + key = self.keys["title"] + self.data[key]["value"] = title # Add the id - self.data['id'] = "urn:ngsi-ld:Catalogue:" + dataset_id + self.data["id"] = "urn:ngsi-ld:Catalogue:" + dataset_id # Add the publisher - key = self.get_key(requested_key='dcterms:publisher') + key = self.get_key(requested_key="dcterms:publisher") position = data.index(key) + 1 - self.data['publisher']['value'] = data[position][0] + self.data["publisher"]["value"] = data[position][0] # Check if we have 'issued' in the original, then we need to create the releaseDate property - index, key, value = get_property_value(data=data, property_name='issued') + index, key, value = get_property_value(data=data, property_name="issued") if index != -1: # We found an 'issued' data - self.data.update(self.__generate_property__(key='releaseDate', value=value[0][0])) + self.data.update( + self.__generate_property__(key="releaseDate", value=value[0][0]) + ) # Get the rest of the data, qb:structure has the same value of qb:dataset, so we decide to # use only qb:dataset in CatalogueDCAT-AP - data = get_rest_data(data=data, not_allowed_keys=['label', 'publisher', 'structure', 'issued', 'title']) + data = get_rest_data( + data=data, + not_allowed_keys=["label", "publisher", "structure", "issued", "title"], + ) # add the new data to the dataset structure self.patch_data(data, False) @@ -136,29 +117,36 @@ def patch_data(self, data, language_map): else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] + [ + self.data.update(self.__generate_property__(key=k, value=v)) + for k, v in data.items() + ] def __complete_label__(self, title, data): try: - key = self.get_key(requested_key='rdfs:label') + key = self.get_key(requested_key="rdfs:label") position = data.index(key) + 1 description = data[position] - descriptions = [x[0].replace("\"", "") for x in description] + descriptions = [x[0].replace('"', "") for x in description] languages = list() try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The Catalogue {title} has a ' - f'rdfs:label without language tag: {description}') + logger.warning( + f"The Catalogue {title} has a " + f"rdfs:label without language tag: {description}" + ) aux = len(description) if aux != 1: - logger.error(f"Catalogue: there is more than 1 description ({aux}), values: {description}") + logger.error( + f"Catalogue: there is more than 1 description ({aux}), values: {description}" + ) else: # There is no language tag, we use by default 'en' - languages = ['en'] + languages = ["en"] logger.warning('Catalogue: selecting default language "en"') ############################################################################### @@ -169,20 +157,20 @@ def __complete_label__(self, title, data): # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - key = self.keys['description'] - self.data[key]['value'][languages[i]] = descriptions[i] + key = self.keys["description"] + self.data[key]["value"][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['language'] - self.data[key]['value'] = languages + key = self.keys["language"] + self.data[key]["value"] = languages except ValueError: - logger.info(f'Dataset without rdfs:label detail: {title}') + logger.info(f"Dataset without rdfs:label detail: {title}") def get(self): return self.data def get_id(self): - return self.data['id'] + return self.data["id"] def get_key(self, requested_key): try: diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index b4dddd1..71fc1c0 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -30,24 +30,14 @@ class Concept(CommonClass): def __init__(self): - super().__init__(entity='Concept') + super().__init__(entity="Concept") self.data = { "id": str(), "type": "Concept", - "language": { - "type": "Property", - "value": list() - }, - "inScheme": { - "type": "Relationship", - "object": str() - }, - "rdfs:subClassOf": { - "type": "Property", - "value": str() - }, - + "language": {"type": "Property", "value": list()}, + "inScheme": {"type": "Relationship", "object": str()}, + "rdfs:subClassOf": {"type": "Property", "value": str()}, ################################################# # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker @@ -57,14 +47,10 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "prefLabel": { - "type": "Property", - "value": dict() - }, - + "prefLabel": {"type": "Property", "value": dict()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } self.concept_id = str() @@ -78,29 +64,35 @@ def add_data(self, concept_id, data): self.concept_id = concept_id try: - position = data.index('skos:prefLabel') + 1 + position = data.index("skos:prefLabel") + 1 except ValueError: # We could not find skos:prefLabel, try to find rdfs:label - position = data.index('rdfs:label') + 1 - logger.warning(f'The Concept {concept_id} does not contain skos:prefLabel but rdfs:label. We use its ' - f'content to fill in the skos:prefLabel property') + position = data.index("rdfs:label") + 1 + logger.warning( + f"The Concept {concept_id} does not contain skos:prefLabel but rdfs:label. We use its " + f"content to fill in the skos:prefLabel property" + ) description = data[position] - descriptions = [x[0].replace("\"", "") for x in description] + descriptions = [x[0].replace('"', "") for x in description] languages = list() try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The Concept {concept_id} has a ' - f'skos:prefLabel without language tag: {description}') + logger.warning( + f"The Concept {concept_id} has a " + f"skos:prefLabel without language tag: {description}" + ) aux = len(description) if aux != 1: - logger.error(f"Concept: there is more than 1 description ({aux}), values: {description}") + logger.error( + f"Concept: there is more than 1 description ({aux}), values: {description}" + ) else: # There is no language tag, we use by default 'en' - languages = ['en'] + languages = ["en"] logger.warning('Concept: selecting default language "en"') # Complete the skos:prefLabel @@ -112,14 +104,14 @@ def add_data(self, concept_id, data): # self.data['skos:prefLabel']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['prefLabel']['value'][languages[i]] = descriptions[i] + self.data["prefLabel"]["value"][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['language'] - self.data[key]['value'] = languages + key = self.keys["language"] + self.data[key]["value"] = languages # Add the id - self.data['id'] = "urn:ngsi-ld:Concept:" + concept_id + self.data["id"] = "urn:ngsi-ld:Concept:" + concept_id # rdfs:seeAlso self.need_add_in_scheme(data=data) @@ -140,49 +132,53 @@ def get(self): return self.data def get_id(self): - return self.data['id'] + return self.data["id"] def need_add_subclass(self, data): try: - position = data.index('rdfs:subClassOf') + 1 - self.data['rdfs:subClassOf']['value'] = data[position][0] + position = data.index("rdfs:subClassOf") + 1 + self.data["rdfs:subClassOf"]["value"] = data[position][0] except ValueError: - logger.info(f'The Concept {self.concept_id} has no rdfs:subClassOf property, deleting the key in the data') + logger.info( + f"The Concept {self.concept_id} has no rdfs:subClassOf property, deleting the key in the data" + ) # We delete the "rdfs:subClassOf" property from the final structure - self.data.pop('rdfs:subClassOf') + self.data.pop("rdfs:subClassOf") def need_add_in_scheme(self, data): position = 0 try: - position = data.index('rdfs:seeAlso') + 1 + position = data.index("rdfs:seeAlso") + 1 except ValueError: # We will try to find the skos:inScheme try: - position = data.index('skos:inScheme') + 1 + position = data.index("skos:inScheme") + 1 except ValueError: - logger.info(f'The Concept {self.concept_id} has neither rdfs:seeAlso or skos:inScheme properties, ' - f'deleting the key in the data') + logger.info( + f"The Concept {self.concept_id} has neither rdfs:seeAlso or skos:inScheme properties, " + f"deleting the key in the data" + ) # We delete the "skos:inScheme" property from the final structure - self.data.pop('skos:inScheme') + self.data.pop("skos:inScheme") parser = RegParser() concept_schema = data[position][0] concept_schema = "urn:ngsi-ld:ConceptSchema:" + parser.obtain_id(concept_schema) - if self.data['inScheme']['type'] == 'Relationship': - self.data['inScheme']['object'] = concept_schema + if self.data["inScheme"]["type"] == "Relationship": + self.data["inScheme"]["object"] = concept_schema else: - self.data['inScheme']['value'] = concept_schema + self.data["inScheme"]["value"] = concept_schema def need_add_notation(self, data): try: - position = data.index('skos:notation') + 1 + position = data.index("skos:notation") + 1 - self.data['notation'] = { - 'type': 'Property', - 'value': data[position][0][0].replace("\"", "") + self.data["notation"] = { + "type": "Property", + "value": data[position][0][0].replace('"', ""), } except ValueError: - logger.info(f'The Concept {self.concept_id} has no skos:notation property') + logger.info(f"The Concept {self.concept_id} has no skos:notation property") diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index 0ca4b32..df21d04 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -30,21 +30,13 @@ class ConceptSchema(CommonClass): def __init__(self): - super().__init__(entity='ConceptScheme') + super().__init__(entity="ConceptScheme") self.data = { "id": str(), "type": "ConceptScheme", - "language": { - "type": "Property", - "value": list() - }, - "hasTopConcept": { - "type": "Relationship", - "object": list() - }, - - + "language": {"type": "Property", "value": list()}, + "hasTopConcept": {"type": "Relationship", "object": list()}, ################################################# # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker @@ -54,15 +46,10 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "prefLabel": { - "type": "Property", - "value": dict() - }, - - + "prefLabel": {"type": "Property", "value": dict()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } self.keys = {k: k for k in self.data.keys()} @@ -70,24 +57,28 @@ def __init__(self): def add_data(self, concept_schema_id, data): # TODO: We have to control that data include the indexes that we want to search # We need to complete the data corresponding to the ConceptSchema: skos:prefLabel - position = data.index('skos:prefLabel') + 1 + position = data.index("skos:prefLabel") + 1 description = data[position] - descriptions = [x[0].replace("\"", "") for x in description] + descriptions = [x[0].replace('"', "") for x in description] languages = list() try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The ConceptSchema {concept_schema_id} has a ' - f'skos:prefLabel without language tag: {description}') + logger.warning( + f"The ConceptSchema {concept_schema_id} has a " + f"skos:prefLabel without language tag: {description}" + ) aux = len(description) if aux != 1: - logger.error(f"ConceptSchema: there is more than 1 description ({aux}), values: {description}") + logger.error( + f"ConceptSchema: there is more than 1 description ({aux}), values: {description}" + ) else: # There is no language tag, we use by default 'en' - languages = ['en'] + languages = ["en"] logger.warning('ConceptSchema: selecting default language "en"') # Complete the skos:prefLabel @@ -99,39 +90,45 @@ def add_data(self, concept_schema_id, data): # self.data['skos:prefLabel']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['prefLabel']['value'][languages[i]] = descriptions[i] + self.data["prefLabel"]["value"][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['language'] - self.data[key]['value'] = languages + key = self.keys["language"] + self.data[key]["value"] = languages # Add the id - self.data['id'] = "urn:ngsi-ld:ConceptSchema:" + concept_schema_id + self.data["id"] = "urn:ngsi-ld:ConceptSchema:" + concept_schema_id # TODO: We need to control that the concept id extracted here are the same that we analyse afterwards. # skos:hasTopConcept, this is a list of ids - position = data.index('skos:hasTopConcept') + 1 - result = list(map(lambda x: self.generate_id(value=x, entity='Concept'), data[position])) - self.data['hasTopConcept']['object'] = result + position = data.index("skos:hasTopConcept") + 1 + result = list( + map(lambda x: self.generate_id(value=x, entity="Concept"), data[position]) + ) + self.data["hasTopConcept"]["object"] = result # Get the rest of data, dct:created and dct:modified properties try: - position = data.index('dct:created') + 1 - self.data['created'] = { + position = data.index("dct:created") + 1 + self.data["created"] = { "type": "Property", - "value": flatten_value(data[position]) + "value": flatten_value(data[position]), } except ValueError: - logger.warning(f'dct:created is not present in the Concept Schema: {concept_schema_id}') + logger.warning( + f"dct:created is not present in the Concept Schema: {concept_schema_id}" + ) try: - position = data.index('dct:modified') + 1 - self.data['modified'] = { + position = data.index("dct:modified") + 1 + self.data["modified"] = { "type": "Property", - "value": flatten_value(data[position]) + "value": flatten_value(data[position]), } except ValueError: - logger.warning(f'dct:modified is not present in the Concept Schema: {concept_schema_id}') + logger.warning( + f"dct:modified is not present in the Concept Schema: {concept_schema_id}" + ) # Order the keys in the final json-ld a = Context() diff --git a/sdmx2jsonld/transform/context.py b/sdmx2jsonld/transform/context.py index e0e0fe5..c19e3e5 100644 --- a/sdmx2jsonld/transform/context.py +++ b/sdmx2jsonld/transform/context.py @@ -20,17 +20,16 @@ # under the License. ## + class Context: def __init__(self): - self.context = { - "@context": dict() - } + self.context = {"@context": dict()} self.fixed_context = [ - 'https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld', - 'http://www.w3.org/ns/dcat#', - 'http://data.europa.eu/(xyz)/statdcat-ap/', - 'http://purl.org/dc/terms/' + "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "http://www.w3.org/ns/dcat#", + "http://data.europa.eu/(xyz)/statdcat-ap/", + "http://purl.org/dc/terms/", ] # Dictionary to keep those contexts that are update from the core contexts @@ -38,18 +37,20 @@ def __init__(self): self.data = dict() # By default, the context should include the smart data models context - self.context['@context']\ - .update({'sdmp': 'https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld'}) + self.context["@context"].update( + { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld" + } + ) # statDCAT-AP contexts - self.context['@context']\ - .update({'dcat': 'http://www.w3.org/ns/dcat#'}) + self.context["@context"].update({"dcat": "http://www.w3.org/ns/dcat#"}) - self.context['@context']\ - .update({'dct': 'http://purl.org/dc/terms/'}) + self.context["@context"].update({"dct": "http://purl.org/dc/terms/"}) - self.context['@context']\ - .update({'stat': 'http://data.europa.eu/(xyz)/statdcat-ap/'}) + self.context["@context"].update( + {"stat": "http://data.europa.eu/(xyz)/statdcat-ap/"} + ) def add_context(self, context): aux = list(context.items()) @@ -57,21 +58,21 @@ def add_context(self, context): value = aux[0][1] found = False - k = '' + k = "" # check if the value of the new_context is in one of the values of the previous context - for k, v in self.context['@context'].items(): + for k, v in self.context["@context"].items(): if v == value: found = True break if not found: # we did not find a key -> New context, we need to add - self.context['@context'].update(context) + self.context["@context"].update(context) else: # We found then we need to change the key in the context or add new one and delete the old one - self.context['@context'].pop(k) - self.context['@context'].update(context) + self.context["@context"].pop(k) + self.context["@context"].update(context) self.context_mapping.update({k: key}) def get_context(self): @@ -85,35 +86,35 @@ def print_context(self): def key_used(self): def key(json_property): - aux = json_property.split(':') + aux = json_property.split(":") if len(aux) == 2: aux = aux[0] else: - aux = '' + aux = "" return aux # Get the list of keys except id, type, and @context keys = list(self.data.keys()) - keys.remove('id') - keys.remove('type') - keys.remove('@context') + keys.remove("id") + keys.remove("type") + keys.remove("@context") prefix_ids = list(map(lambda x: key(json_property=x), keys)) - prefix_ids = list(filter(lambda x: x != '', prefix_ids)) + prefix_ids = list(filter(lambda x: x != "", prefix_ids)) prefix_ids = [*set(prefix_ids)] return prefix_ids def reduce_context(self, used_keys): # 1st: Get the key-value of the fixed context - aux = dict((new_val, new_k) for new_k, new_val in self.data['@context'].items()) + aux = dict((new_val, new_k) for new_k, new_val in self.data["@context"].items()) aux = list(map(lambda x: aux[x], self.fixed_context)) # 2nd: Join fixed_context and used_keys aux = aux + used_keys # 3rd: Get the new context - new_context = list(map(lambda x: {x: self.data['@context'][x]}, aux)) + new_context = list(map(lambda x: {x: self.data["@context"][x]}, aux)) new_context = dict((key, val) for k in new_context for key, val in k.items()) # TODO: we should fix if the rest of context lines are needed or they are duplicated some properties @@ -131,18 +132,18 @@ def new_analysis(self): used_keys = self.key_used() new_context = self.reduce_context(used_keys=used_keys) - self.data['@context'] = new_context + self.data["@context"] = new_context def order_context(self): # I want that the content of the dict, 1st id, 2nd type, last context # Get all the keys and initialize the order keys = list(self.data.keys()) - keys.remove('type') - keys.remove('id') - keys.remove('@context') + keys.remove("type") + keys.remove("id") + keys.remove("@context") # initializing order - ord_list = ['id', 'type'] + keys + ['@context'] + ord_list = ["id", "type"] + keys + ["@context"] # Custom order dictionary # Using dictionary comprehension @@ -155,27 +156,43 @@ def set_data(self, new_data): self.data = new_data -if __name__ == '__main__': +if __name__ == "__main__": a = Context() a.print_context() - a.add_context({'rdf': ''}) + a.add_context({"rdf": ""}) a.print_context() print() - data = {'type': 'Dataset', 'id': 'urn:ngsi-ld:Dataset:dsd3001', - 'dc:title': 'http://bauhaus/structuresDeDonnees/structure/dsd3001', 'dc:identifier': 'dsd3001', - 'dc:language': {'type': 'Property', 'value': ['en', 'fr']}, - 'dc:description': {'type': 'Property', 'value': {'en': 'SDMX DSD NA_MAIN', 'fr': 'SDMX NA_MAIN'}}, - '@context': {'sdmp': 'https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld', - 'dcat': 'http://www.w3.org/ns/dcat#', 'stat': 'http://data.europa.eu/(xyz)/statdcat-ap/', - 'qb': 'http://purl.org/linked-data/cube#', 'dc11': 'http://purl.org/dc/elements/1.1/', - 'dc': 'http://purl.org/dc/terms/', 'xsd': 'http://www.w3.org/2001/XMLSchema#', - 'ns0': 'http://rdf.insee.fr/def/base#', 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#', - 'skos': 'http://www.w3.org/2004/02/skos/core#', 'owl': 'http://www.w3.org/2002/07/owl#'}, - 'dc11:contributor': 'DG75-H250', 'dc11:creator': 'DG57-L201', 'dc:created': '2022-01-15T08:00:00+00:00', - 'dc:modified': '2022-01-15T10:00:00+00:00', - 'other_thing': 'foo'} + data = { + "type": "Dataset", + "id": "urn:ngsi-ld:Dataset:dsd3001", + "dc:title": "http://bauhaus/structuresDeDonnees/structure/dsd3001", + "dc:identifier": "dsd3001", + "dc:language": {"type": "Property", "value": ["en", "fr"]}, + "dc:description": { + "type": "Property", + "value": {"en": "SDMX DSD NA_MAIN", "fr": "SDMX NA_MAIN"}, + }, + "@context": { + "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld", + "dcat": "http://www.w3.org/ns/dcat#", + "stat": "http://data.europa.eu/(xyz)/statdcat-ap/", + "qb": "http://purl.org/linked-data/cube#", + "dc11": "http://purl.org/dc/elements/1.1/", + "dc": "http://purl.org/dc/terms/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "ns0": "http://rdf.insee.fr/def/base#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "owl": "http://www.w3.org/2002/07/owl#", + }, + "dc11:contributor": "DG75-H250", + "dc11:creator": "DG57-L201", + "dc:created": "2022-01-15T08:00:00+00:00", + "dc:modified": "2022-01-15T10:00:00+00:00", + "other_thing": "foo", + } a.set_data(new_data=data) a.new_analysis() diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 98e5370..14f618a 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -64,30 +64,34 @@ class Dataset(CommonClass): def __init__(self): - super().__init__(entity='Dataset') + super().__init__(entity="Dataset") # TODO: These dimensions are not defined in the turtle file but defined in a prefix therefore at the moment # we create manually their corresponding DimensionProperty entity. Should we generated from checking the prefix - self.list_special_components = ['freq', 'refArea', 'timePeriod', - 'obsStatus', 'confStatus', 'timeFormat', - 'timePerCollect', 'decimals', 'title', - 'unitMult', 'compilingOrg', 'dataComp', - 'currency', 'dissOrg', 'measureType'] + self.list_special_components = [ + "freq", + "refArea", + "timePeriod", + "obsStatus", + "confStatus", + "timeFormat", + "timePerCollect", + "decimals", + "title", + "unitMult", + "compilingOrg", + "dataComp", + "currency", + "dissOrg", + "measureType", + ] self.data = { "id": str(), "type": "Dataset", - "title": { - "type": "Property", - "value": str() - }, + "title": {"type": "Property", "value": str()}, "identifier": str(), - "language": { - "type": "Property", - "value": list() - }, - - + "language": {"type": "Property", "value": list()}, ################################################# # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker @@ -97,60 +101,56 @@ def __init__(self): # "LanguageMap": dict() # }, ################################################# - "description": { - "type": "Property", - "value": dict() - }, - - + "description": {"type": "Property", "value": dict()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } self.components = { - 'qb:attribute': { - 'entity': 'AttributeProperty', - 'key': 'attribute', - 'value': { - "attribute": { - "type": "Relationship", - "object": list() - } - } + "qb:attribute": { + "entity": "AttributeProperty", + "key": "attribute", + "value": {"attribute": {"type": "Relationship", "object": list()}}, }, - 'qb:dimension': { - 'entity': 'DimensionProperty', - 'key': 'dimension', - 'value': { - "dimension": { - "type": "Relationship", - "object": list() - } - } + "qb:dimension": { + "entity": "DimensionProperty", + "key": "dimension", + "value": {"dimension": {"type": "Relationship", "object": list()}}, + }, + "qb:measure": { + "entity": "statUnitMeasure", + "key": "statUnitMeasure", + "value": { + "statUnitMeasure": {"type": "Relationship", "object": list()} + }, }, - 'qb:measure': { - 'entity': 'statUnitMeasure', - 'key': 'statUnitMeasure', - 'value': { - "statUnitMeasure": { - "type": "Relationship", - "object": list() - } - } - } } - self.keys = {k: k for k in self.data.keys()} | \ - {self.components['qb:attribute']['key']: self.components['qb:attribute']['key']} | \ - {self.components['qb:dimension']['key']: self.components['qb:dimension']['key']} | \ - {self.components['qb:measure']['key']: self.components['qb:measure']['key']} + self.keys = ( + {k: k for k in self.data.keys()} + | { + self.components["qb:attribute"]["key"]: self.components["qb:attribute"][ + "key" + ] + } + | { + self.components["qb:dimension"]["key"]: self.components["qb:dimension"][ + "key" + ] + } + | { + self.components["qb:measure"]["key"]: self.components["qb:measure"][ + "key" + ] + } + ) self.sdmx_dimensions = { "freq": Frequency(), "refArea": RefArea(), "timePeriod": TimePeriod(), - "measureType": MeasureType() + "measureType": MeasureType(), } self.sdmx_attributes = { @@ -164,12 +164,12 @@ def __init__(self): "compilingOrg": CompilingOrg(), "dataComp": DataComp(), "dissOrg": DissOrg(), - "currency": Currency() + "currency": Currency(), } self.sdmx_components = { "DimensionProperty": self.sdmx_dimensions, - "AttributeProperty": self.sdmx_attributes + "AttributeProperty": self.sdmx_attributes, } self.sdmx_concepts = { @@ -187,7 +187,7 @@ def __init__(self): "dataComp": DataCompConcept(), "dissOrg": DissOrgConcept(), "currency": CurrencyConcept(), - "measureType": None + "measureType": None, } self.sdmx_concept_schemas = CogConceptSchema() @@ -195,48 +195,56 @@ def __init__(self): def add_components(self, component): # We need to know which kind of component we have, it should be the verb: # qb:attribute, qb:dimension, or qb:measure - list_components = ['qb:attribute', 'qb:dimension', 'qb:measure'] + list_components = ["qb:attribute", "qb:dimension", "qb:measure"] type_component = [x for x in list_components if x in component][0] position = component.index(type_component) + 1 if type_component == "qb:measure": - logger.info(f'The qb:measure "{component[position][0]}" is not manage in statDCAT-AP') + logger.info( + f'The qb:measure "{component[position][0]}" is not manage in statDCAT-AP' + ) new_component, new_concept, new_concept_schema = None, None, None else: - new_component, new_concept, new_concept_schema = self.manage_components(type_component=type_component, - component=component, - position=position) + new_component, new_concept, new_concept_schema = self.manage_components( + type_component=type_component, component=component, position=position + ) return new_component, new_concept, new_concept_schema def manage_components(self, type_component, component, position): new_component, new_concept, new_concept_schema = None, None, None try: - entity = self.components[type_component]['entity'] - name, new_id = self.generate_id(entity=entity, value=component[position][0], update_id=False) - key = self.components[type_component]['key'] + entity = self.components[type_component]["entity"] + name, new_id = self.generate_id( + entity=entity, value=component[position][0], update_id=False + ) + key = self.components[type_component]["key"] # It is possible that the original file contains already the description - if new_id in self.components[type_component]['value'][key]['object']: + if new_id in self.components[type_component]["value"][key]["object"]: logger.warning( - f"The component {new_id} is duplicated and already defined in the {self.data['id']}") + f"The component {new_id} is duplicated and already defined in the {self.data['id']}" + ) elif name in self.list_special_components: # We need to create manually the description of these dimensions, concepts, and conceptschemas logger.warning( f"The component {name} is defined probably outside of the file, " - f"creating manually the {entity} entity") - self.components[type_component]['value'][key]['object'].append(new_id) - self.data = self.data | self.components[type_component]['value'] + f"creating manually the {entity} entity" + ) + self.components[type_component]["value"][key]["object"].append(new_id) + self.data = self.data | self.components[type_component]["value"] new_component = self.sdmx_components[entity][name] new_concept = self.sdmx_concepts[name] new_concept_schema = self.sdmx_concept_schemas else: - self.components[type_component]['value'][key]['object'].append(new_id) - self.data = self.data | self.components[type_component]['value'] + self.components[type_component]["value"][key]["object"].append(new_id) + self.data = self.data | self.components[type_component]["value"] except ValueError: - logger.error(f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}") + logger.error( + f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}" + ) # Order the keys in the final json-ld a = Context() @@ -254,26 +262,25 @@ def add_data(self, title, dataset_id, data): self.__complete_label__(title=title, data=data) # Add the title - key = self.keys['title'] - self.data[key]['value'] = title + key = self.keys["title"] + self.data[key]["value"] = title # Add the id - self.data['id'] = "urn:ngsi-ld:Dataset:" + dataset_id + self.data["id"] = "urn:ngsi-ld:Dataset:" + dataset_id # Get the rest of the data - data = get_rest_data(data=data, - not_allowed_keys=[ - 'sliceKey', - 'component', - 'disseminationStatus', - 'validationState', - 'notation', - 'label' - ], - further_process_keys=[ - 'component', - 'label' - ]) + data = get_rest_data( + data=data, + not_allowed_keys=[ + "sliceKey", + "component", + "disseminationStatus", + "validationState", + "notation", + "label", + ], + further_process_keys=["component", "label"], + ) # add the new data to the dataset structure self.patch_data(data, False) @@ -284,29 +291,36 @@ def patch_data(self, data, language_map): else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] + [ + self.data.update(self.__generate_property__(key=k, value=v)) + for k, v in data.items() + ] def __complete_label__(self, title, data): try: - key = self.get_key(requested_key='rdfs:label') + key = self.get_key(requested_key="rdfs:label") position = data.index(key) + 1 description = data[position] - descriptions = [x[0].replace("\"", "") for x in description] + descriptions = [x[0].replace('"', "") for x in description] languages = list() try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The Dataset {title} has a ' - f'rdfs:label without language tag: {description}') + logger.warning( + f"The Dataset {title} has a " + f"rdfs:label without language tag: {description}" + ) aux = len(description) if aux != 1: - logger.error(f"Dataset: there is more than 1 description ({aux}), values: {description}") + logger.error( + f"Dataset: there is more than 1 description ({aux}), values: {description}" + ) else: # There is no language tag, we use by default 'en' - languages = ['en'] + languages = ["en"] logger.warning('Dataset: selecting default language "en"') ############################################################################### @@ -317,14 +331,14 @@ def __complete_label__(self, title, data): # self.data['rdfs:label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - key = self.keys['description'] - self.data[key]['value'][languages[i]] = descriptions[i] + key = self.keys["description"] + self.data[key]["value"][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['language'] - self.data[key]['value'] = languages + key = self.keys["language"] + self.data[key]["value"] = languages except ValueError: - logger.info(f'DataStructureDefinition without rdfs:label detail: {title}') + logger.info(f"DataStructureDefinition without rdfs:label detail: {title}") def get_key(self, requested_key): try: diff --git a/sdmx2jsonld/transform/dimension.py b/sdmx2jsonld/transform/dimension.py index de03424..850f3e0 100644 --- a/sdmx2jsonld/transform/dimension.py +++ b/sdmx2jsonld/transform/dimension.py @@ -25,11 +25,11 @@ class Dimension(Property): def __init__(self): - super().__init__(entity='DimensionProperty') - self.data['type'] = 'DimensionProperty' + super().__init__(entity="DimensionProperty") + self.data["type"] = "DimensionProperty" def add_data(self, property_id, data): super().add_data(property_id=property_id, data=data) # Add the id - self.data['id'] = "urn:ngsi-ld:DimensionProperty:" + property_id + self.data["id"] = "urn:ngsi-ld:DimensionProperty:" + property_id diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index 64a5a2c..398f344 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -38,49 +38,39 @@ def __init__(self): "type": "Property", "value": [ "/ngsi-ld/v1/entities?type=https://smartdatamodels.org/dataModel.SDMX/Observation" - ] + ], }, "description": { "type": "Property", - "value": "Distribution of statistical data observations." - }, - "format": { - "type": "Property", - "value": "JSON_LD" - }, - "language": { - "type": "Property", - "value": list() - }, - "status": { - "type": "Property", - "value": "Completed" - }, - "title": { - "type": "Property", - "value": list() + "value": "Distribution of statistical data observations.", }, + "format": {"type": "Property", "value": "JSON_LD"}, + "language": {"type": "Property", "value": list()}, + "status": {"type": "Property", "value": "Completed"}, + "title": {"type": "Property", "value": list()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } def generate_data(self, catalogue): # Generate random id for the distribution random_bits = getrandbits(128) hash1 = "%032x" % random_bits - self.data['id'] += hash1 + self.data["id"] += hash1 # Title is extracted from the dcterms:title from the Catalogue - self.data['title']['value'] = catalogue.data['title']['value'] + self.data["title"]["value"] = catalogue.data["title"]["value"] # language es obtained from language from the Catalogue - self.data['language']['value'] = catalogue.data['language']['value'] + self.data["language"]["value"] = catalogue.data["language"]["value"] # accessURL is generated from the configuration file. config_path = dirname(dirname(dirname(__file__))) - config_path = join(join(config_path, 'common'), 'config.json') + config_path = join(join(config_path, "common"), "config.json") with open(config_path) as config_file: config = load(config_file) - self.data['accessUrl']['value'][0] = config['broker'] + self.data['accessUrl']['value'][0] + self.data["accessUrl"]["value"][0] = ( + config["broker"] + self.data["accessUrl"]["value"][0] + ) diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 7b8d225..4ed1102 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -30,7 +30,11 @@ from logging import getLogger from datetime import datetime from sdmx2jsonld.common.regparser import RegParser -from sdmx2jsonld.common.classprecedence import Precedence, ClassPrecedencePropertyError, ClassPrecedenceClassError +from sdmx2jsonld.common.classprecedence import ( + Precedence, + ClassPrecedencePropertyError, + ClassPrecedenceClassError, +) logger = getLogger() @@ -38,17 +42,17 @@ class EntityType: def __init__(self): self.entities = { - 'qb:DataSet': 'Catalogue', - 'qb:Observation': 'Observation', - 'qb:DataStructureDefinition': 'Dataset', - 'qb:ComponentSpecification': 'Component', - 'qb:AttributeProperty': 'Attribute', - 'qb:DimensionProperty': 'Dimension', - 'qb:CodedProperty': 'Dimension', - 'rdfs:Class': 'Class', - 'owl:Class': 'Class', - 'skos:ConceptScheme': 'ConceptScheme', - 'skos:Concept': 'Range' + "qb:DataSet": "Catalogue", + "qb:Observation": "Observation", + "qb:DataStructureDefinition": "Dataset", + "qb:ComponentSpecification": "Component", + "qb:AttributeProperty": "Attribute", + "qb:DimensionProperty": "Dimension", + "qb:CodedProperty": "Dimension", + "rdfs:Class": "Class", + "owl:Class": "Class", + "skos:ConceptScheme": "ConceptScheme", + "skos:Concept": "Range", } self.dataset = Dataset() @@ -77,8 +81,8 @@ def __find_entity_type__(self, string): # in case that there is no verb, we are talking about a triples whose id was previously # created. try: - position = string1.index('a') + 1 - data = '' + position = string1.index("a") + 1 + data = "" try: data = self.pre.precedence(string1[position]) @@ -91,17 +95,19 @@ def __find_entity_type__(self, string): data = self.entities[data[0]] except ClassPrecedenceClassError as error: logger.warning(str(error)) - data = self.entities['rdfs:Class'] + data = self.entities["rdfs:Class"] except KeyError: # We found a CodeList or any other thing, check the list of codeList found in the turtle file if data not in self.conceptListsIds: logger.warning(f"Received a unexpected entity type: {data}") else: - data = 'Range' + data = "Range" is_new = True except ValueError: - logger.info(f'Not a definition triples {string}, need to find the proper structure') + logger.info( + f"Not a definition triples {string}, need to find the proper structure" + ) is_new = False data = self.__get_subject__(title=string[0]) string1 = string[1:] @@ -114,7 +120,9 @@ def transform(self, string): if is_new: self.create_data(entity_type=data_type, data=new_string, title=string[0]) else: - logger.info(f'Checking previous subjects to find if it was created previously') + logger.info( + f"Checking previous subjects to find if it was created previously" + ) self.patch_data(datatype=data_type, data=new_string) def patch_data(self, datatype, data): @@ -124,35 +132,43 @@ def flatten_value(y): elif isinstance(y, datetime): return y else: - return y.replace('"', '') + return y.replace('"', "") flatten_data = [item for sublist in data for item in sublist] - if flatten_data[0] != 'rdfs:label': - flatten_data = {flatten_data[i]: flatten_value(flatten_data[i + 1]) for i in range(0, len(flatten_data), 2)} + if flatten_data[0] != "rdfs:label": + flatten_data = { + flatten_data[i]: flatten_value(flatten_data[i + 1]) + for i in range(0, len(flatten_data), 2) + } language_map = False else: language_map = True - if datatype == 'Dataset': + if datatype == "Dataset": self.dataset.patch_data(data=flatten_data, language_map=language_map) def create_data(self, entity_type, data, title): - if entity_type == 'Component': - some_new_component, some_new_concept, some_new_concept_schema = \ - self.dataset.add_components(component=data) + if entity_type == "Component": + ( + some_new_component, + some_new_concept, + some_new_concept_schema, + ) = self.dataset.add_components(component=data) if some_new_component is not None: - if some_new_component.data['type'] == 'DimensionProperty': + if some_new_component.data["type"] == "DimensionProperty": # we have found special sdmx_dimensions that we have to add to dimensions list self.dimensions.append(some_new_component) - elif some_new_component.data['type'] == 'AttributeProperty': + elif some_new_component.data["type"] == "AttributeProperty": # we have found special sdmx_attribute that we have to add to attributes list self.attributes.append(some_new_component) else: # You should not be here, reporting error... - logger.error(f'Unexpected entity type, id: {some_new_component.data["id"]} ' - f'type: {some_new_component.data["type"]}') + logger.error( + f'Unexpected entity type, id: {some_new_component.data["id"]} ' + f'type: {some_new_component.data["type"]}' + ) if some_new_concept is not None: self.conceptLists.append(some_new_concept) @@ -160,43 +176,43 @@ def create_data(self, entity_type, data, title): # we need to check that the conceptSchema is not already defined in the structure if some_new_concept_schema not in self.conceptSchemas: self.conceptSchemas.append(some_new_concept_schema) - elif entity_type == 'Catalogue': + elif entity_type == "Catalogue": identifier = self.parser.obtain_id(title) self.catalogue.add_data(title=title, dataset_id=identifier, data=data) - elif entity_type == 'Observation': + elif entity_type == "Observation": observation = Observation() identifier = self.parser.obtain_id(title) observation.add_data(title=title, observation_id=identifier, data=data) self.observations.append(observation) - elif entity_type == 'Dataset': + elif entity_type == "Dataset": identifier = self.parser.obtain_id(title) self.dataset.add_data(title=title, dataset_id=identifier, data=data) # Create the CatalogueDCAT-AP and assign the dataset id - self.catalogue.add_dataset(dataset_id=self.dataset.data['id']) - elif entity_type == 'Dimension': + self.catalogue.add_dataset(dataset_id=self.dataset.data["id"]) + elif entity_type == "Dimension": dimension = Dimension() dimension_id = self.parser.obtain_id(title) dimension.add_data(property_id=dimension_id, data=data) self.dimensions.append(dimension) - elif entity_type == 'Attribute': + elif entity_type == "Attribute": attribute = Attribute() attribute_id = self.parser.obtain_id(title) attribute.add_data(attribute_id=attribute_id, data=data) self.attributes.append(attribute) - elif entity_type == 'ConceptScheme': + elif entity_type == "ConceptScheme": concept_schema = ConceptSchema() concept_schema_id = self.parser.obtain_id(title) concept_schema.add_data(concept_schema_id=concept_schema_id, data=data) self.conceptSchemas.append(concept_schema) - elif entity_type == 'Class': + elif entity_type == "Class": # We need the Concept because each of the Range description is of the type Concept concept_list = Concept() concept_list_id = self.parser.obtain_id(title) concept_list.add_data(concept_id=concept_list_id, data=data) self.conceptLists.append(concept_list) self.conceptListsIds[title] = concept_list.get_id() - elif entity_type == 'Range': + elif entity_type == "Range": # TODO: Range is associated to a Concept and identified properly in the ConceptSchema data_range = Concept() data_range_id = self.parser.obtain_id(title) @@ -207,8 +223,8 @@ def create_data(self, entity_type, data, title): logger.error(f'Entity type "{entity_type}" not processed.') def __get_subject__(self, title): - if self.dataset.get()['dct:title'] == title: - return 'Dataset' + if self.dataset.get()["dct:title"] == title: + return "Dataset" else: AssertionError(f"Still not defined: {title}") diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index 867a280..eb8fd72 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -34,145 +34,133 @@ class Observation(CommonClass): def __init__(self): - super().__init__(entity='Observation') + super().__init__(entity="Observation") self.data = { "id": str(), "type": "Observation", - "title": { - "type": "Property", - "value": str() - }, - "identifier": { - "type": "Property", - "value": str() - }, - "dataSet": { - "type": "Relationship", - "object": str() - }, - "confStatus": { - "type": "Property", - "value": str() - }, - "decimals": { - "type": "Property", - "value": int() - }, - "obsStatus": { - "type": "Property", - "value": str() - }, - "unitMult": { - "type": "Property", - "value": int() - }, - "freq": { - "type": "Property", - "value": str() - }, - "refArea": { - "type": "Property", - "value": str() - }, - "timePeriod": { - "type": "Property", - "value": str() - }, - "obsValue": { - "type": "Property", - "value": float() - }, - "dimensions": { - "type": "Property", - "value": list() - }, + "title": {"type": "Property", "value": str()}, + "identifier": {"type": "Property", "value": str()}, + "dataSet": {"type": "Relationship", "object": str()}, + "confStatus": {"type": "Property", "value": str()}, + "decimals": {"type": "Property", "value": int()}, + "obsStatus": {"type": "Property", "value": str()}, + "unitMult": {"type": "Property", "value": int()}, + "freq": {"type": "Property", "value": str()}, + "refArea": {"type": "Property", "value": str()}, + "timePeriod": {"type": "Property", "value": str()}, + "obsValue": {"type": "Property", "value": float()}, + "dimensions": {"type": "Property", "value": list()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.SDMX/master/context.jsonld" - ] + ], } self.concept_id = str() self.keys = {k: k for k in self.data.keys()} - self.regexpDimension = compile('ns1:.*') + self.regexpDimension = compile("ns1:.*") def add_data(self, title, observation_id, data): # We have a list of dimensions, a dataset, a list of attributes, a list of dimensions attributes # and an observation # Add the confStatus - key = self.__assign_property__(requested_key='sdmx-attribute:confStatus', data=data) - self.data[key]['value'] = ConfStatus().fix_value(value=self.data[key]['value']) + key = self.__assign_property__( + requested_key="sdmx-attribute:confStatus", data=data + ) + self.data[key]["value"] = ConfStatus().fix_value(value=self.data[key]["value"]) # Add the id - self.data['id'] = "urn:ngsi-ld:Observation:" + observation_id - self.data['identifier']['value'] = observation_id + self.data["id"] = "urn:ngsi-ld:Observation:" + observation_id + self.data["identifier"]["value"] = observation_id # Add title, the url string as it is in the turtle document - self.data['title']['value'] = title + self.data["title"]["value"] = title # Add the decimals - key = self.__assign_property__(requested_key='sdmx-attribute:decimals', data=data) - self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) + key = self.__assign_property__( + requested_key="sdmx-attribute:decimals", data=data + ) + self.data[key]["value"] = Code(typecode=key).fix_value( + value=self.data[key]["value"] + ) # Add obsStatus - key = self.__assign_property__(requested_key='sdmx-attribute:obsStatus', data=data) - self.data[key]['value'] = ObsStatus().fix_value(value=self.data[key]['value']) + key = self.__assign_property__( + requested_key="sdmx-attribute:obsStatus", data=data + ) + self.data[key]["value"] = ObsStatus().fix_value(value=self.data[key]["value"]) # Add unitMult - key = self.__assign_property__(requested_key='sdmx-attribute:unitMult', data=data) - self.data[key]['value'] = Code(typecode=key).fix_value(value=self.data[key]['value']) + key = self.__assign_property__( + requested_key="sdmx-attribute:unitMult", data=data + ) + self.data[key]["value"] = Code(typecode=key).fix_value( + value=self.data[key]["value"] + ) # Add freq "pattern": "^_[OUZ]|[SQBNI]|OA|OM|[AMWDH]_*[0-9]*$" # TODO: Add verification of coded data following the pattern - key = self.__assign_property__(requested_key='sdmx-dimension:freq', data=data) - self.data[key]['value'] = Frequency().fix_value(value=self.data[key]['value']) + key = self.__assign_property__(requested_key="sdmx-dimension:freq", data=data) + self.data[key]["value"] = Frequency().fix_value(value=self.data[key]["value"]) # Add reference Area # TODO: Add verification of coded data following ISO-2, ISO-3, or M49 code - _ = self.__assign_property__(requested_key='sdmx-dimension:refArea', data=data) + _ = self.__assign_property__(requested_key="sdmx-dimension:refArea", data=data) # Add timePeriod - _ = self.__assign_property__(requested_key='sdmx-dimension:timePeriod', data=data) + _ = self.__assign_property__( + requested_key="sdmx-dimension:timePeriod", data=data + ) # Add obsValue - _ = self.__assign_property__(requested_key='sdmx-measure:obsValue', data=data) + _ = self.__assign_property__(requested_key="sdmx-measure:obsValue", data=data) # Add dataset parser = RegParser() - key = self.__assign_property__(requested_key='qb:dataSet', data=data, key_property='object') - identifier = parser.obtain_id(self.data[key]['object']) - self.data[key]['object'] = 'urn:ngsi-ld:CatalogueDCAT-AP:' + identifier + key = self.__assign_property__( + requested_key="qb:dataSet", data=data, key_property="object" + ) + identifier = parser.obtain_id(self.data[key]["object"]) + self.data[key]["object"] = "urn:ngsi-ld:CatalogueDCAT-AP:" + identifier # Add dimensions result = self.__assign_dimensions__(data=data) - self.data['dimensions']['value'] = result + self.data["dimensions"]["value"] = result def __assign_dimensions__(self, data): def create_data(key, value): - new_dimension = { - "key": key, - "value": value - } + new_dimension = {"key": key, "value": value} return new_dimension + parser = RegParser() # We need to get the list of available dimension keys - dimension_keys = [a for a in data if self.regexpDimension.match(a.__str__()) is not None] + dimension_keys = [ + a for a in data if self.regexpDimension.match(a.__str__()) is not None + ] # We need to get the list of values for these keys, if there is an url, it is considered a values = [self.get_data(data[data.index(a) + 1]) for a in dimension_keys] - values = [parser.obtain_id(a, prefix_string="urn:ngsi-ld:Concept:") for a in values] + values = [ + parser.obtain_id(a, prefix_string="urn:ngsi-ld:Concept:") for a in values + ] # Get the list of Entity IDs - entity_ids = ["urn:ngsi-ld:DimensionProperty:" + b.split("ns1:", 1)[1] for b in dimension_keys] + entity_ids = [ + "urn:ngsi-ld:DimensionProperty:" + b.split("ns1:", 1)[1] + for b in dimension_keys + ] - result = [create_data(key=entity_ids[i], value=values[i]) for i in range(0, len(values))] + result = [ + create_data(key=entity_ids[i], value=values[i]) + for i in range(0, len(values)) + ] return result - def __assign_property__(self, requested_key, data, key_property='value'): + def __assign_property__(self, requested_key, data, key_property="value"): key = self.get_key(requested_key=requested_key) position = data.index(requested_key) + 1 self.data[key][key_property] = self.get_data(data[position]) @@ -186,7 +174,7 @@ def get_key(self, requested_key): except KeyError: # The key did not exist therefore we add to the list with this value # We need to check if it exists without prefix - m = search('(.*):(.*)', requested_key) + m = search("(.*):(.*)", requested_key) if m is not None: subfix = m.group(2) diff --git a/sdmx2jsonld/transform/parser.py b/sdmx2jsonld/transform/parser.py index 2c4e3c5..ef740ba 100644 --- a/sdmx2jsonld/transform/parser.py +++ b/sdmx2jsonld/transform/parser.py @@ -33,7 +33,7 @@ logger = getLogger(__name__) -__version__ = "0.5.2" +__version__ = "1.0.0" class Parser: @@ -42,7 +42,7 @@ def __init__(self): with open(GRAMMARFILE) as f: grammar = f.read() - self.parser = Lark(grammar, start='start', parser='lalr') + self.parser = Lark(grammar, start="start", parser="lalr") def parsing(self, content: TextIOBase, out: bool = False): """ @@ -83,7 +83,7 @@ def parsing_file(self, content: TextIOWrapper, out: bool): if out: # Save the generated content into files - logger.info('Save the generated content into files') + logger.info("Save the generated content into files") transform.save() elif content is not None: print() @@ -93,14 +93,14 @@ def parsing_file(self, content: TextIOWrapper, out: bool): ds = transform.get_dataset() if ds is not None: self.__check_pprint__(transform.get_dataset()) - [pprint(x.get()) for x in transform.get_dimensions()] - [pprint(x.get()) for x in transform.get_attributes()] - [pprint(x.get()) for x in transform.get_concept_schemas()] - [pprint(x.get()) for x in transform.get_concept_lists()] + [pprint(x.get()) for x in transform.get_dimensions()] # type: ignore[func-returns-value] + [pprint(x.get()) for x in transform.get_attributes()] # type: ignore[func-returns-value] + [pprint(x.get()) for x in transform.get_concept_schemas()] # type: ignore[func-returns-value] + [pprint(x.get()) for x in transform.get_concept_lists()] # type: ignore[func-returns-value] observations = transform.entity_type.get_observation() if len(observations) != 0: - [pprint(x.get()) for x in observations] + [pprint(x.get()) for x in observations] # type: ignore[func-returns-value] # If we have several observations, we need to generate the DCAT-AP:Distribution class distribution = Distribution() @@ -128,19 +128,17 @@ def parsing_string(self, content: StringIO): result.append(ds) dimensions = transform.get_dimensions() - [result.append(x.get()) for x in dimensions] + [result.append(x.get()) for x in dimensions] # type: ignore[func-returns-value] - [result.append(x.get()) for x in transform.get_attributes()] + [result.append(x.get()) for x in transform.get_attributes()] # type: ignore[func-returns-value] - [result.append(x.get()) for x in transform.get_concept_schemas()] + [result.append(x.get()) for x in transform.get_concept_schemas()] # type: ignore[func-returns-value] - [result.append(x.get()) for x in transform.get_concept_lists()] - # jicg - [result.append(x.get()) for x in transform.get_observation()] + [result.append(x.get()) for x in transform.get_concept_lists()] # type: ignore[func-returns-value] observations = transform.entity_type.get_observation() - # jicg - observations = transform.get_observation() if len(observations) != 0: - [result.append(x.get()) for x in observations] + [result.append(x.get()) for x in observations] # type: ignore[func-returns-value] # If we have several observations, we need to generate the DCAT-AP:Distribution class distribution = Distribution() diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index caf479b..c5907ce 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -35,11 +35,7 @@ def __init__(self, entity): self.data = { "id": str(), "type": "", - "language": { - "type": "Property", - "value": list() - }, - + "language": {"type": "Property", "value": list()}, ################################################# # TODO: New ETSI CIM NGSI-LD specification 1.4.2 # Pending to implement in the Context Broker @@ -49,22 +45,12 @@ def __init__(self, entity): # "LanguageMap": dict() # }, ################################################# - "label": { - "type": "Property", - "value": dict() - }, - - "codeList": { - "type": "Relationship", - "object": str() - }, - "concept": { - "type": "Relationship", - "object": str() - }, + "label": {"type": "Property", "value": dict()}, + "codeList": {"type": "Relationship", "object": str()}, + "concept": {"type": "Relationship", "object": str()}, "@context": [ "https://raw.githubusercontent.com/smart-data-models/dataModel.STAT-DCAT-AP/master/context.jsonld" - ] + ], } self.keys = {k: k for k in self.data.keys()} @@ -72,24 +58,28 @@ def __init__(self, entity): def add_data(self, property_id, data): # TODO: We have to control that data include the indexes that we want to search # We need to complete the data corresponding to the Dimension: rdfs:label - position = data.index('rdfs:label') + 1 + position = data.index("rdfs:label") + 1 description = data[position] - descriptions = [x[0].replace("\"", "") for x in description] + descriptions = [x[0].replace('"', "") for x in description] languages = list() try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning(f'The Property {property_id} has a ' - f'rdfs:label without language tag: {description}') + logger.warning( + f"The Property {property_id} has a " + f"rdfs:label without language tag: {description}" + ) aux = len(description) if aux != 1: - logger.error(f"Property: there is more than 1 description ({aux}), values: {description}") + logger.error( + f"Property: there is more than 1 description ({aux}), values: {description}" + ) else: # There is no language tag, we use by default 'en' - languages = ['en'] + languages = ["en"] logger.warning('Property: selecting default language "en"') ############################################################################### @@ -100,49 +90,55 @@ def add_data(self, property_id, data): # self.data['label']['LanguageMap'][languages[i]] = descriptions[i] ############################################################################### for i in range(0, len(languages)): - self.data['label']['value'][languages[i]] = descriptions[i] + self.data["label"]["value"][languages[i]] = descriptions[i] # Complete the information of the language with the previous information - key = self.keys['language'] - self.data[key]['value'] = languages + key = self.keys["language"] + self.data[key]["value"] = languages # qb:codeList, this attribute might not be presented, so we need to check it. # TODO: We need to control that the codeList id extracted here are the same that we analyse afterwards. try: - position = data.index('qb:codeList') + 1 - code_list = self.generate_id(entity="ConceptSchema", value=data[position][0]) - self.data['codeList']['object'] = code_list + position = data.index("qb:codeList") + 1 + code_list = self.generate_id( + entity="ConceptSchema", value=data[position][0] + ) + self.data["codeList"]["object"] = code_list except ValueError: - logger.warning(f'Property: {property_id} has not qb:codeList, deleting the key in the data') + logger.warning( + f"Property: {property_id} has not qb:codeList, deleting the key in the data" + ) # If we have not the property, we delete it from data - self.data.pop('codeList') + self.data.pop("codeList") # qb:concept # TODO: the concept id need to check if it is a normal id or an url - position = data.index('qb:concept') + 1 + position = data.index("qb:concept") + 1 concept = self.generate_id(entity="Concept", value=data[position][0]) - self.data['concept']['object'] = concept + self.data["concept"]["object"] = concept # Get the rest of the data - data = get_rest_data(data=data, - not_allowed_keys=[ - 'sliceKey', - 'component', - 'disseminationStatus', - 'validationState', - 'notation', - 'label', - 'codeList', - 'concept' - ], - further_process_keys=[ - 'component', - 'label' - ]) + data = get_rest_data( + data=data, + not_allowed_keys=[ + "sliceKey", + "component", + "disseminationStatus", + "validationState", + "notation", + "label", + "codeList", + "concept", + ], + further_process_keys=["component", "label"], + ) # add the new data to the dataset structure - [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] + [ + self.data.update(self.__generate_property__(key=k, value=v)) + for k, v in data.items() + ] # Order the keys in the final json-ld a = Context() diff --git a/sdmx2jsonld/transform/transformer.py b/sdmx2jsonld/transform/transformer.py index 354aaa2..021ba8a 100644 --- a/sdmx2jsonld/transform/transformer.py +++ b/sdmx2jsonld/transform/transformer.py @@ -50,7 +50,7 @@ def triples(self, triple): return triple def predicate(self, pre): - result = '' + result = "" if isinstance(pre[0], str): result = pre[0] else: @@ -115,11 +115,11 @@ def get_catalogue(self): return self.entity_type.get_catalogue() def get_observation(self): - if self.entity_type.observations.data['id'] != '': + if self.entity_type.observations.data["id"] != "": return self.entity_type.get_observation() def get_dataset(self): - if self.entity_type.dataset.data['id'] != '': + if self.entity_type.dataset.data["id"] != "": return self.entity_type.get_dataset() return None @@ -136,10 +136,10 @@ def get_concept_lists(self): return self.entity_type.get_concept_list() def save(self): - self.entity_type.save('catalogue') + self.entity_type.save("catalogue") - if self.entity_type.dataset.data['id'] != '': - self.entity_type.save('dataset') + if self.entity_type.dataset.data["id"] != "": + self.entity_type.save("dataset") dimensions = self.entity_type.get_dimensions() [dimension.save() for dimension in dimensions] diff --git a/tests/common/test_classprecedence.py b/tests/common/test_classprecedence.py index a9347ef..f433139 100644 --- a/tests/common/test_classprecedence.py +++ b/tests/common/test_classprecedence.py @@ -20,7 +20,11 @@ # under the License. ## from unittest import TestCase -from sdmx2jsonld.common.classprecedence import Precedence, ClassPrecedencePropertyError, ClassPrecedenceClassError +from sdmx2jsonld.common.classprecedence import ( + Precedence, + ClassPrecedencePropertyError, + ClassPrecedenceClassError, +) class Test(TestCase): @@ -31,44 +35,52 @@ def test_precedence_one_class(self): """ The precedence of one Class will be ALWAYS that Class """ - obtained = self.pre.precedence(['qb:DataStructureDefinition']) - expected = 'qb:DataStructureDefinition' - assert obtained == expected, f"'qb:DataStructureDefinition' expected, got: '{obtained}'" - - obtained = self.pre.precedence(['skos:Concept']) - expected = 'skos:Concept' + obtained = self.pre.precedence(["qb:DataStructureDefinition"]) + expected = "qb:DataStructureDefinition" + assert ( + obtained == expected + ), f"'qb:DataStructureDefinition' expected, got: '{obtained}'" + + obtained = self.pre.precedence(["skos:Concept"]) + expected = "skos:Concept" assert obtained == expected, f"'skos:Concept' expected, got: '{obtained}'" - obtained = self.pre.precedence(['qb:SliceKey']) - expected = 'qb:SliceKey' + obtained = self.pre.precedence(["qb:SliceKey"]) + expected = "qb:SliceKey" assert obtained == expected, f"'qb:SliceKey' expected, got: '{obtained}'" - obtained = self.pre.precedence(['qb:DimensionProperty']) - expected = 'qb:DimensionProperty' - assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" + obtained = self.pre.precedence(["qb:DimensionProperty"]) + expected = "qb:DimensionProperty" + assert ( + obtained == expected + ), f"'qb:DimensionProperty' expected, got: '{obtained}'" - obtained = self.pre.precedence(['qb:AttributeProperty']) - expected = 'qb:AttributeProperty' - assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" + obtained = self.pre.precedence(["qb:AttributeProperty"]) + expected = "qb:AttributeProperty" + assert ( + obtained == expected + ), f"'qb:AttributeProperty' expected, got: '{obtained}'" - obtained = self.pre.precedence(['skos:ConceptScheme']) - expected = 'skos:ConceptScheme' + obtained = self.pre.precedence(["skos:ConceptScheme"]) + expected = "skos:ConceptScheme" assert obtained == expected, f"'skos:ConceptScheme' expected, got: '{obtained}'" - obtained = self.pre.precedence(['owl:Class']) - expected = 'owl:Class' + obtained = self.pre.precedence(["owl:Class"]) + expected = "owl:Class" assert obtained == expected, f"'owl:Class' expected, got: '{obtained}'" - obtained = self.pre.precedence(['qb:ComponentSpecification']) - expected = 'qb:ComponentSpecification' - assert obtained == expected, f"'qb:ComponentSpecification' expected, got: '{obtained}'" + obtained = self.pre.precedence(["qb:ComponentSpecification"]) + expected = "qb:ComponentSpecification" + assert ( + obtained == expected + ), f"'qb:ComponentSpecification' expected, got: '{obtained}'" - obtained = self.pre.precedence(['qb:MeasureProperty']) - expected = 'qb:MeasureProperty' + obtained = self.pre.precedence(["qb:MeasureProperty"]) + expected = "qb:MeasureProperty" assert obtained == expected, f"'qb:MeasureProperty' expected, got: '{obtained}'" - obtained = self.pre.precedence(['skos:ConceptScheme']) - expected = 'skos:ConceptScheme' + obtained = self.pre.precedence(["skos:ConceptScheme"]) + expected = "skos:ConceptScheme" assert obtained == expected, f"'skos:ConceptScheme' expected, got: '{obtained}'" def test_precedence_classes_with_dimension_and_attribute_values(self): @@ -81,52 +93,73 @@ def test_precedence_classes_with_dimension_and_attribute_values(self): with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:DimensionProperty", "qb:AttributeProperty"]) - self.assertEqual(str(error.exception), - "['qb:DimensionProperty', 'qb:AttributeProperty'] -> Incompatible multiclass definition") + self.assertEqual( + str(error.exception), + "['qb:DimensionProperty', 'qb:AttributeProperty'] -> Incompatible multiclass definition", + ) def test_precedence_classes_with_attribute_and_measure_values(self): with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:AttributeProperty", "qb:MeasureProperty"]) - self.assertEqual(str(error.exception), - "['qb:AttributeProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition") + self.assertEqual( + str(error.exception), + "['qb:AttributeProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition", + ) def test_precedence_classes_with_dimension_and_measure_values(self): with self.assertRaises(ClassPrecedencePropertyError) as error: _ = self.pre.precedence(["qb:DimensionProperty", "qb:MeasureProperty"]) - self.assertEqual(str(error.exception), - "['qb:DimensionProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition") + self.assertEqual( + str(error.exception), + "['qb:DimensionProperty', 'qb:MeasureProperty'] -> Incompatible multiclass definition", + ) def test_precedence_classes_with_class_values(self): with self.assertRaises(ClassPrecedenceClassError) as error: _ = self.pre.precedence(["rdfs:Class", "owl:Class"]) - self.assertEqual(str(error.exception), "['rdfs:Class', 'owl:Class'] -> Possible redundant Class definition") + self.assertEqual( + str(error.exception), + "['rdfs:Class', 'owl:Class'] -> Possible redundant Class definition", + ) def test_attribute_and_coded_property(self): - obtained = self.pre.precedence(['qb:AttributeProperty', 'qb:CodedProperty']) - expected = 'qb:AttributeProperty' - assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" - - obtained = self.pre.precedence(['qb:CodedProperty', 'qb:AttributeProperty']) - expected = 'qb:AttributeProperty' - assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" + obtained = self.pre.precedence(["qb:AttributeProperty", "qb:CodedProperty"]) + expected = "qb:AttributeProperty" + assert ( + obtained == expected + ), f"'qb:AttributeProperty' expected, got: '{obtained}'" + + obtained = self.pre.precedence(["qb:CodedProperty", "qb:AttributeProperty"]) + expected = "qb:AttributeProperty" + assert ( + obtained == expected + ), f"'qb:AttributeProperty' expected, got: '{obtained}'" def test_coded_and_dimension_property(self): - obtained = self.pre.precedence(['qb:CodedProperty', 'qb:DimensionProperty']) - expected = 'qb:DimensionProperty' - assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" - - obtained = self.pre.precedence(['qb:DimensionProperty', 'qb:CodedProperty']) - expected = 'qb:DimensionProperty' - assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" + obtained = self.pre.precedence(["qb:CodedProperty", "qb:DimensionProperty"]) + expected = "qb:DimensionProperty" + assert ( + obtained == expected + ), f"'qb:DimensionProperty' expected, got: '{obtained}'" + + obtained = self.pre.precedence(["qb:DimensionProperty", "qb:CodedProperty"]) + expected = "qb:DimensionProperty" + assert ( + obtained == expected + ), f"'qb:DimensionProperty' expected, got: '{obtained}'" def test_concept_and_other_property(self): - obtained = self.pre.precedence(['skos:Concept', '']) - expected = 'skos:Concept' + obtained = self.pre.precedence( + ["skos:Concept", ""] + ) + expected = "skos:Concept" assert obtained == expected, f"'skos:Concept' expected, got: '{obtained}'" - obtained = self.pre.precedence(['', 'skos:Concept']) - expected = 'skos:Concept' + obtained = self.pre.precedence( + ["", "skos:Concept"] + ) + expected = "skos:Concept" assert obtained == expected, f"'skos:Concept' expected, got: '{obtained}'" diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py index 7810a53..0896b9b 100644 --- a/tests/common/test_commonclass.py +++ b/tests/common/test_commonclass.py @@ -25,25 +25,34 @@ import os import json + class TestCommonClass(TestCase): def setUp(self) -> None: pass def test_instance_class(self): cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) - assert (urnid == "urn:ngsi-ld:test.common.entity:entity_id") + urnid = cclass.generate_id( + "https://string-to-parse-ur/entity_id", update_id=True + ) + assert urnid == "urn:ngsi-ld:test.common.entity:entity_id" # urnid = cclass.generate_id("") # print(urnid) def test_save(self): - context = {"@context": "https://raw.githubusercontent.com/smart-data-models/data-models/master/context/merge_subjects_config_example.json"} - context_map = {"address": "https://smartdatamodels.org/address", - "alternateName": "https://smartdatamodels.org/alternateName", - "status": "ngsi-ld:status"} + context = { + "@context": "https://raw.githubusercontent.com/smart-data-models/data-models/master/context/merge_subjects_config_example.json" + } + context_map = { + "address": "https://smartdatamodels.org/address", + "alternateName": "https://smartdatamodels.org/alternateName", + "status": "ngsi-ld:status", + } cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) - assert(urnid == "urn:ngsi-ld:test.common.entity:entity_id") + urnid = cclass.generate_id( + "https://string-to-parse-ur/entity_id", update_id=True + ) + assert urnid == "urn:ngsi-ld:test.common.entity:entity_id" cclass.add_context(context, context_map) @@ -51,10 +60,11 @@ def test_save(self): os.chdir("/tmp/commonclass") cclass.save() - with open("/tmp/commonclass/output/test.common.entity_entity_id.jsonld", "r") as f: + with open( + "/tmp/commonclass/output/test.common.entity_entity_id.jsonld", "r" + ) as f: data = json.load(f) - assert(data['id'] == urnid) - assert(data['@context'] == context['@context']) + assert data["id"] == urnid + assert data["@context"] == context["@context"] # TODO - Add tests with cclass.generate_id using update_id with a False value - diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index 8f13e9b..5726d54 100755 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -30,64 +30,94 @@ def setUp(self) -> None: def test_string_to_integer(self): dtc = DataTypeConversion() - token_type = 'xsd:int' + token_type = "xsd:int" # token = Token('FORMATCONNECTOR', "432112") i: int = dtc.convert("432112", token_type) - assert (i == 432112) + assert i == 432112 i = dtc.convert("-100", token_type) - assert (i == -100) + assert i == -100 i = dtc.convert("19223372036854775808", token_type) - assert (i == 19223372036854775808) + assert i == 19223372036854775808 assert type(i) is int try: dtc.convert("invalid value", token_type) - assert (False) + assert False except Exception: - assert (True) + assert True def test_string_to_bool(self): dtc = DataTypeConversion() - token_type = 'xsd:boolean' + token_type = "xsd:boolean" values = ("True", "true", "y", "yes", "T", "1", 1, True) for value in values: - assert (dtc.convert(f'"{value}"', token_type)) - - values = ("fAlsE", "False", "N", "No", "F", "0", "0.0", "", "None", None, [], {}, 0, 0.0) + assert dtc.convert(f'"{value}"', token_type) + + values = ( + "fAlsE", + "False", + "N", + "No", + "F", + "0", + "0.0", + "", + "None", + None, + [], + {}, + 0, + 0.0, + ) for value in values: - assert (not dtc.convert(f'"{value}"', token_type)) + assert not dtc.convert(f'"{value}"', token_type) invalid_values = (5, 4.2, "invalid value", "nil") for value in invalid_values: try: dtc.convert(value, token_type) - assert (False) + assert False except Exception: - assert (True) + assert True def test_string_to_dates(self): dtc = DataTypeConversion() - token_type = 'xsd:dateTime' - - dates = ('"2022-01-15T08:00:00.000+00:00"', '"2022-01-10T09:00:00.000"', - '"2021-07-01T11:50:37.3"', '"2021-09-28T15:31:24.05"', - '"Mon Jan 13 09:52:52 MST 2014"', '"Thu Jun 02 11:56:53 CDT 2011"', - '"2022-12-12T10:00:00"', '"2022-05-11T10:00:00"', - '"Tue Dec 13 11:00:00 K 2022"', '"2021-07-01T11:58:08.642000"') - expected = ("2022-01-15T08:00:00+00:00", "2022-01-10T08:00:00+00:00", - "2021-07-01T09:50:37.300000+00:00", "2021-09-28T13:31:24.050000+00:00", - "2014-01-13T16:52:52+00:00", "2011-06-02T16:56:53+00:00", - "2022-12-12T09:00:00+00:00", "2022-05-11T08:00:00+00:00", - "2022-12-13T01:00:00+00:00", "2021-07-01T09:58:08.642000+00:00") - # "2021-07-01T09:58:08.642000+00:00" + token_type = "xsd:dateTime" + + dates = ( + '"2022-01-15T08:00:00.000+00:00"', + '"2022-01-10T09:00:00.000"', + '"2021-07-01T11:50:37.3"', + '"2021-09-28T15:31:24.05"', + '"Mon Jan 13 09:52:52 MST 2014"', + '"Thu Jun 02 11:56:53 CDT 2011"', + '"2022-12-12T10:00:00"', + '"2022-05-11T10:00:00"', + '"Tue Dec 13 11:00:00 K 2022"', + '"2021-07-01T11:58:08.642000"', + ) + expected = ( + "2022-01-15T08:00:00+00:00", + "2022-01-10T08:00:00+00:00", + "2021-07-01T09:50:37.300000+00:00", + "2021-09-28T13:31:24.050000+00:00", + "2014-01-13T16:52:52+00:00", + "2011-06-02T16:56:53+00:00", + "2022-12-12T09:00:00+00:00", + "2022-05-11T08:00:00+00:00", + "2022-12-13T01:00:00+00:00", + "2021-07-01T09:58:08.642000+00:00", + ) + # "2021-07-01T09:58:08.642000+00:00" d = zip(dates, expected) for test_date, expected_date in d: - assert (expected_date == dtc.convert(test_date, token_type)) + assert expected_date == dtc.convert(test_date, token_type) + class Test(TestCase): def setUp(self): @@ -97,31 +127,39 @@ def test_datetime_string_conversion_1(self): """ Check if we can get a correct datetime value from a string, case 1 """ - obtained = self.conversion.convert('"2022-01-15T08:00:00.000 UTC"', 'xsd:dateTime') - expected = '2022-01-15T08:00:00+00:00' - assert obtained == expected, f"\n\nDateTime was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + obtained = self.conversion.convert( + '"2022-01-15T08:00:00.000 UTC"', "xsd:dateTime" + ) + expected = "2022-01-15T08:00:00+00:00" + assert obtained == expected, ( + f"\n\nDateTime was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_int_string_conversion(self): """ Check if we can get a correct integer from a string """ - obtained = self.conversion.convert('"2"', 'xsd:int') + obtained = self.conversion.convert('"2"', "xsd:int") expected = 2 - assert obtained == expected, f"\n\nInteger was not the expected," \ - f"\n got : {obtained} {type(obtained)}" \ - f"\n expected: {expected} {type(expected)}" + assert obtained == expected, ( + f"\n\nInteger was not the expected," + f"\n got : {obtained} {type(obtained)}" + f"\n expected: {expected} {type(expected)}" + ) def test_int_integer_conversion(self): """ Check if we can get a correct integer from an integer """ - obtained = self.conversion.convert('2', 'xsd:int') + obtained = self.conversion.convert("2", "xsd:int") expected = 2 - assert obtained == expected, f"\n\nInteger was not the expected," \ - f"\n got : {obtained} {type(obtained)}" \ - f"\n expected: {expected} {type(expected)}" + assert obtained == expected, ( + f"\n\nInteger was not the expected," + f"\n got : {obtained} {type(obtained)}" + f"\n expected: {expected} {type(expected)}" + ) # # print(dataConversionType.convert(data23[0], data23[2]) + 10) # @@ -130,21 +168,22 @@ def test_boolean_conversion(self): """ Check if we can convert a boolean string into its proper value """ - obtained = self.conversion.convert('"true"', 'xsd:boolean') + obtained = self.conversion.convert('"true"', "xsd:boolean") expected = True - assert obtained == expected, f"\n\nBoolean was not the expected," \ - f"\n got : {obtained} {type(obtained)}" \ - f"\n expected: {expected} {type(expected)}" + assert obtained == expected, ( + f"\n\nBoolean was not the expected," + f"\n got : {obtained} {type(obtained)}" + f"\n expected: {expected} {type(expected)}" + ) def test_fake_conversion(self): """ Check is a fake value data launch an exception """ with self.assertRaises(Exception) as error: - _ = self.conversion.convert('"fake"', 'otraCosa') + _ = self.conversion.convert('"fake"', "otraCosa") - self.assertEqual(str(error.exception), - "Datatype not defined: otraCosa") + self.assertEqual(str(error.exception), "Datatype not defined: otraCosa") # # Convert datetime generated into UTC format: 2021-12-21T16:18:55Z or 2021-12-21T16:18:55+00:00, ISO8601 # @@ -162,18 +201,22 @@ def test_float_float_conversion(self): """ Check if we can get a correct integer from an string """ - obtained = self.conversion.convert('2345.2', 'xsd:float') + obtained = self.conversion.convert("2345.2", "xsd:float") expected = 2345.2 - assert obtained == expected, f"\n\nInteger was not the expected," \ - f"\n got : {obtained} {type(obtained)}" \ - f"\n expected: {expected} {type(expected)}" + assert obtained == expected, ( + f"\n\nInteger was not the expected," + f"\n got : {obtained} {type(obtained)}" + f"\n expected: {expected} {type(expected)}" + ) def test_float_string_conversion(self): """ Check if we can get a correct integer from an integer """ - obtained = self.conversion.convert('"3016.9"', 'xsd:float') + obtained = self.conversion.convert('"3016.9"', "xsd:float") expected = 3016.9 - assert obtained == expected, f"\n\nFloat was not the expected," \ - f"\n got : {obtained} {type(obtained)}" \ - f"\n expected: {expected} {type(expected)}" + assert obtained == expected, ( + f"\n\nFloat was not the expected," + f"\n got : {obtained} {type(obtained)}" + f"\n expected: {expected} {type(expected)}" + ) diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index 46bd2c9..2d5d003 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -20,7 +20,12 @@ # under the License. ## from unittest import TestCase -from sdmx2jsonld.common.listmanagement import get_rest_data, flatten_value, extract_prefix, get_property_value +from sdmx2jsonld.common.listmanagement import ( + get_rest_data, + flatten_value, + extract_prefix, + get_property_value, +) from sdmx2jsonld.exceptions.exceptions import ClassExtractPrefixError @@ -29,28 +34,66 @@ def setUp(self) -> None: pass def test_get_rest_data(self): - data = ['a', ['qb:AttributeProperty'], 'rdfs:label', [['"SDMX attribute COMMENT_OBS"', '@en'], ['"Attribut SDMX "', '@fr']], 'dct:created', [['2022-01-15T06:00:00+00:00']], 'dct:identifier', [['"a3003"']], 'dct:modified', [['2022-01-15T06:30:00+00:00']], 'qb:concept', ['http://bauhaus/concepts/definition/c4303'], 'insee:disseminationStatus', ['http://id.insee.fr/codes/base/statutDiffusion/Prive'], 'insee:validationState', [['"Unpublished"']], 'rdfs:range', ['xsd:string'], 'skos:notation', [['"COMMENT_OBS"']]] - not_allowed_keys = ['sliceKey', 'component', 'disseminationStatus', 'validationState', 'notation', 'label', 'codeList', 'concept'] - further_process_keys = ['component', 'label'] - expected_res = {'created': '2022-01-15T06:00:00+00:00', 'identifier': 'a3003', 'modified': '2022-01-15T06:30:00+00:00', 'range': 'xsd:string'} + data = [ + "a", + ["qb:AttributeProperty"], + "rdfs:label", + [['"SDMX attribute COMMENT_OBS"', "@en"], ['"Attribut SDMX "', "@fr"]], + "dct:created", + [["2022-01-15T06:00:00+00:00"]], + "dct:identifier", + [['"a3003"']], + "dct:modified", + [["2022-01-15T06:30:00+00:00"]], + "qb:concept", + ["http://bauhaus/concepts/definition/c4303"], + "insee:disseminationStatus", + ["http://id.insee.fr/codes/base/statutDiffusion/Prive"], + "insee:validationState", + [['"Unpublished"']], + "rdfs:range", + ["xsd:string"], + "skos:notation", + [['"COMMENT_OBS"']], + ] + not_allowed_keys = [ + "sliceKey", + "component", + "disseminationStatus", + "validationState", + "notation", + "label", + "codeList", + "concept", + ] + further_process_keys = ["component", "label"] + expected_res = { + "created": "2022-01-15T06:00:00+00:00", + "identifier": "a3003", + "modified": "2022-01-15T06:30:00+00:00", + "range": "xsd:string", + } res = get_rest_data(data, not_allowed_keys, further_process_keys) - assert(expected_res == res) + assert expected_res == res def test_flatten_value(self): - data = [['"SDMX attribute PRE_BREAK_VALUE"', '@en'], ['"Attribut SDMX "', '@fr']] - expected_res = {'en': 'SDMX attribute PRE_BREAK_VALUE', 'fr': 'Attribut SDMX '} + data = [ + ['"SDMX attribute PRE_BREAK_VALUE"', "@en"], + ['"Attribut SDMX "', "@fr"], + ] + expected_res = {"en": "SDMX attribute PRE_BREAK_VALUE", "fr": "Attribut SDMX "} got_data = flatten_value(data) - assert(got_data == expected_res) + assert got_data == expected_res # data = ['', None, 'dct:created', 'dct:identifier', 'dct:modified', 'rdfs:range', 'uno', 'a:b:c'] def test_extract_prefix_with_a_prefix(self): - data = 'a:b' - expected_res = 'b' + data = "a:b" + expected_res = "b" got_res = extract_prefix(data) - assert(got_res == expected_res) + assert got_res == expected_res def test_extract_prefix_with_several_prefixes(self): - data = 'a:b:c' + data = "a:b:c" expected = "Unexpected number of prefixes: 'a:b:c'" with self.assertRaises(ClassExtractPrefixError) as error: _ = extract_prefix(attribute=data) @@ -66,7 +109,7 @@ def test_extract_prefix_with_None_value(self): self.assertEqual(str(error.exception.message), expected) def test_extract_prefix_with_empty_value(self): - data = '' + data = "" expected = "Unexpected data received: ''" with self.assertRaises(ClassExtractPrefixError) as error: _ = extract_prefix(attribute=data) @@ -74,87 +117,117 @@ def test_extract_prefix_with_empty_value(self): self.assertEqual(str(error.exception.message), expected) def test_extract_prefix_with_a_value_without_prefix(self): - data = 'a' - expected_res = 'a' + data = "a" + expected_res = "a" got_res = extract_prefix(data) - assert(got_res == expected_res) + assert got_res == expected_res def test_get_property_data_from_array_property_without_prefix(self): - data = ['a', - ['qb:DataSet'], - 'rdfs:label', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'dcterms:issued', - [['2022-04-01T06:00:00+00:00']], - 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], - 'dcterms:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'qb:structure', - ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], - 'sdmx-attribute:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']] - ] - expected = [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']] - - index, key, obtained = get_property_value(data=data, property_name='title') + data = [ + "a", + ["qb:DataSet"], + "rdfs:label", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "dcterms:issued", + [["2022-04-01T06:00:00+00:00"]], + "dcterms:publisher", + ["http://id.insee.fr/organisations/insee"], + "dcterms:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "qb:structure", + ["http://bauhaus/structuresDeDonnees/structure/dsd3001"], + "sdmx-attribute:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + ] + expected = [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ] + + index, key, obtained = get_property_value(data=data, property_name="title") self.assertEqual(index, 8) - self.assertEqual(key, 'dcterms:title') + self.assertEqual(key, "dcterms:title") self.assertEqual(expected, obtained) def test_get_property_data_from_array_property_with_prefix(self): - data = ['a', - ['qb:DataSet'], - 'rdfs:label', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'dcterms:issued', - [['2022-04-01T06:00:00+00:00']], - 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], - 'dcterms:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'qb:structure', - ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], - 'sdmx-attribute:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']] - ] - expected = [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']] - - index, key, obtained = get_property_value(data=data, property_name='dcterms:title') + data = [ + "a", + ["qb:DataSet"], + "rdfs:label", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "dcterms:issued", + [["2022-04-01T06:00:00+00:00"]], + "dcterms:publisher", + ["http://id.insee.fr/organisations/insee"], + "dcterms:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "qb:structure", + ["http://bauhaus/structuresDeDonnees/structure/dsd3001"], + "sdmx-attribute:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + ] + expected = [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ] + + index, key, obtained = get_property_value( + data=data, property_name="dcterms:title" + ) self.assertEqual(index, 8) - self.assertEqual(key, 'dcterms:title') + self.assertEqual(key, "dcterms:title") self.assertEqual(expected, obtained) def test_get_property_data_from_array_invalid_property(self): - data = ['a', - ['qb:DataSet'], - 'rdfs:label', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'dcterms:issued', - [['2022-04-01T06:00:00+00:00']], - 'dcterms:publisher', ['http://id.insee.fr/organisations/insee'], - 'dcterms:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']], - 'qb:structure', - ['http://bauhaus/structuresDeDonnees/structure/dsd3001'], - 'sdmx-attribute:title', - [['"GDP and main components (current prices)"', '@en'], - ['"PIB et principales composantes (prix courants)"', '@fr']] - ] - expected = '' - - index, key, obtained = get_property_value(data=data, property_name='any') + data = [ + "a", + ["qb:DataSet"], + "rdfs:label", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "dcterms:issued", + [["2022-04-01T06:00:00+00:00"]], + "dcterms:publisher", + ["http://id.insee.fr/organisations/insee"], + "dcterms:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + "qb:structure", + ["http://bauhaus/structuresDeDonnees/structure/dsd3001"], + "sdmx-attribute:title", + [ + ['"GDP and main components (current prices)"', "@en"], + ['"PIB et principales composantes (prix courants)"', "@fr"], + ], + ] + expected = "" + + index, key, obtained = get_property_value(data=data, property_name="any") self.assertEqual(index, -1) - self.assertEqual(key, '') + self.assertEqual(key, "") self.assertEqual(expected, obtained) diff --git a/tests/common/test_rdf.py b/tests/common/test_rdf.py index 12aa53c..25145df 100644 --- a/tests/common/test_rdf.py +++ b/tests/common/test_rdf.py @@ -29,7 +29,7 @@ def setUp(self) -> None: pass def test_turtle_1(self): - rdf_data = ''' + rdf_data = """ @prefix ab: . ab:richard ab:homeTel "(229) 276-5135" . @@ -41,11 +41,11 @@ def test_turtle_1(self): ab:craig ab:homeTel "(194) 966-1505" . ab:craig ab:email "craigellis@yahoo.com" . ab:craig ab:email "c.ellis@usairwaysgroup.com" . - ''' + """ rdf_content = turtle_terse(rdf_data) - + gx = Graph() gx = gx.parse(data=rdf_content, format="turtle") ser = gx.serialize(format="turtle") - assert(rdf_content == ser) + assert rdf_content == ser diff --git a/tests/common/test_regparser.py b/tests/common/test_regparser.py index 5dabb9c..589c859 100644 --- a/tests/common/test_regparser.py +++ b/tests/common/test_regparser.py @@ -29,6 +29,4 @@ def setUp(self) -> None: def test_get_id(self): re = RegParser() - assert(re.obtain_id("https://elmundo.es/episode-one") == "episode-one") - - + assert re.obtain_id("https://elmundo.es/episode-one") == "episode-one" diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py index 05adc50..83201e7 100644 --- a/tests/sdmxattributes/test_code.py +++ b/tests/sdmxattributes/test_code.py @@ -26,18 +26,22 @@ class TestConfStatus(TestCase): def test_code_value_with_prefix(self): - value = 'sdmx-code:decimals-1' + value = "sdmx-code:decimals-1" expected = 1 code = Code(typecode="decimals") obtained = code.fix_value(value=value) - assert obtained == expected, f"\ncode was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\ncode was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_code_negative_value_with_prefix(self): - value = 'sdmx-code:decimals--1' - expected = 'sdmx-code:decimals--1 -> decimals out of range, got: -1 range(0, 15)' + value = "sdmx-code:decimals--1" + expected = ( + "sdmx-code:decimals--1 -> decimals out of range, got: -1 range(0, 15)" + ) code = Code(typecode="decimals") @@ -47,8 +51,10 @@ def test_code_negative_value_with_prefix(self): self.assertEqual(str(error.exception), expected) def test_code_value_bigger_than_maximum_with_prefix(self): - value = 'sdmx-code:decimals-67' - expected = 'sdmx-code:decimals-67 -> decimals out of range, got: 67 range(0, 15)' + value = "sdmx-code:decimals-67" + expected = ( + "sdmx-code:decimals-67 -> decimals out of range, got: 67 range(0, 15)" + ) code = Code(typecode="decimals") @@ -58,18 +64,20 @@ def test_code_value_bigger_than_maximum_with_prefix(self): self.assertEqual(str(error.exception), expected) def test_code_value_without_prefix(self): - value = 'unitMult-1' + value = "unitMult-1" expected = 1 code = Code(typecode="unitMult") obtained = code.fix_value(value=value) - assert obtained == expected, f"\ncode was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\ncode was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_code_negative_value_without_prefix(self): - value = 'unitMult--1' - expected = 'unitMult--1 -> unitMult out of range, got: -1 range(0, 13)' + value = "unitMult--1" + expected = "unitMult--1 -> unitMult out of range, got: -1 range(0, 13)" code = Code(typecode="unitMult") @@ -79,8 +87,8 @@ def test_code_negative_value_without_prefix(self): self.assertEqual(str(error.exception), expected) def test_code_value_bigger_than_maximum_without_prefix(self): - value = 'unitMult-67' - expected = 'unitMult-67 -> unitMult out of range, got: 67 range(0, 13)' + value = "unitMult-67" + expected = "unitMult-67 -> unitMult out of range, got: 67 range(0, 13)" code = Code(typecode="unitMult") @@ -95,13 +103,15 @@ def test_code_integer_value(self): code = Code(typecode="unitMult") obtained = code.fix_value(value=value) - assert obtained == expected, f"\ncode was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\ncode was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_code_integer_value_out_of_range(self): value = 25 - expected = '25 -> unitMult out of range, got: 25 range(0, 13)' + expected = "25 -> unitMult out of range, got: 25 range(0, 13)" code = Code(typecode="unitMult") @@ -111,18 +121,20 @@ def test_code_integer_value_out_of_range(self): self.assertEqual(str(error.exception), expected) def test_code_string_value(self): - value = '2' + value = "2" expected = 2 code = Code(typecode="unitMult") obtained = code.fix_value(value=value) - assert obtained == expected, f"\ncode was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\ncode was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_code_string_value_out_of_range(self): - value = '25' - expected = '25 -> unitMult out of range, got: 25 range(0, 13)' + value = "25" + expected = "25 -> unitMult out of range, got: 25 range(0, 13)" code = Code(typecode="unitMult") @@ -132,8 +144,8 @@ def test_code_string_value_out_of_range(self): self.assertEqual(str(error.exception), expected) def test_any_other_code(self): - value = 'sadf' - expected = 'sadf -> Data is not a valid value' + value = "sadf" + expected = "sadf -> Data is not a valid value" code = Code(typecode="unitMult") diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py index 806abdb..fc24c86 100644 --- a/tests/sdmxattributes/test_confirmationStatus.py +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -29,42 +29,52 @@ def setUp(self): self.conversion = ConfStatus() def test_fix_value_data_in_predefined_values(self): - value = 'F' - expected = 'F' + value = "F" + expected = "F" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nconfStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) - value = 'f' - expected = 'F' + value = "f" + expected = "F" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nconfStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_fix_value_data_in_predefined_values_with_prefix(self): - value = 'confStatus-A' - expected = 'A' + value = "confStatus-A" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nconfStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) - value = 'confstatus-a' - expected = 'A' + value = "confstatus-a" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nconfStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nconfStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): - value = 'confStatus-EEEEE' - expected = "confStatus-EEEEE -> ConfStatus value is not included in the list of available values,\n" + \ - " got:confStatus-EEEEE\n" + \ - " expected:['confStatus-F', 'confStatus-N', 'confStatus-C', 'confStatus-D', 'confStatus-S', " \ - "'confStatus-A', 'confStatus-O', 'confStatus-T', 'confStatus-G', 'confStatus-M', 'confStatus-E', " \ - "'confStatus-P']" + value = "confStatus-EEEEE" + expected = ( + "confStatus-EEEEE -> ConfStatus value is not included in the list of available values,\n" + + " got:confStatus-EEEEE\n" + + " expected:['confStatus-F', 'confStatus-N', 'confStatus-C', 'confStatus-D', 'confStatus-S', " + "'confStatus-A', 'confStatus-O', 'confStatus-T', 'confStatus-G', 'confStatus-M', 'confStatus-E', " + "'confStatus-P']" + ) with self.assertRaises(ClassConfStatusError) as error: _ = self.conversion.fix_value(value=value) @@ -72,16 +82,16 @@ def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): self.assertEqual(str(error.exception), expected) def test_fix_value_unexpected_value_without_prefix(self): - value = 'EEEE' - expected = 'EEEE -> ConfStatus value is not the expected' + value = "EEEE" + expected = "EEEE -> ConfStatus value is not the expected" with self.assertRaises(ClassConfStatusError) as error: _ = self.conversion.fix_value(value=value) self.assertEqual(str(error.exception), expected) def test_fix_value_unexpected_prefix(self): - value = 'lkdjlks-A' - expected = 'lkdjlks-A -> ConfStatus value is not the expected' + value = "lkdjlks-A" + expected = "lkdjlks-A -> ConfStatus value is not the expected" with self.assertRaises(ClassConfStatusError) as error: _ = self.conversion.fix_value(value=value) diff --git a/tests/sdmxattributes/test_examples.py b/tests/sdmxattributes/test_examples.py index 2160fd5..2639354 100644 --- a/tests/sdmxattributes/test_examples.py +++ b/tests/sdmxattributes/test_examples.py @@ -32,10 +32,14 @@ def setUp(self) -> None: examples_folder = dirname(dirname(__file__)) output_folder = dirname(examples_folder) - examples_folder = join(examples_folder, 'files') - self.output_folder = join(output_folder, 'output') - - tests_files = ["observations.ttl", "structures-accounts.ttl", "structures-tourism.ttl"] + examples_folder = join(examples_folder, "files") + self.output_folder = join(output_folder, "output") + + tests_files = [ + "observations.ttl", + "structures-accounts.ttl", + "structures-tourism.ttl", + ] self.tests_files = [join(examples_folder, x) for x in tests_files] self.parser = Parser() @@ -48,7 +52,7 @@ def clean_output_dir(self) -> None: unlink(file_path) except Exception as e: print("----------------------------") - print(e.message) + print(e) print("----------------------------") def test_files_from_StringIO_web_interface_loop(self): @@ -65,9 +69,11 @@ def test_files_from_StringIO_web_interface_loop(self): try: _ = self.parser.parsing(content=StringIO(rdf_data), out=False) except Exception as e: - assert False, f"\nThe parser was not completed," \ - f"\n file: {a}" \ - f"\n exception:\n {e.message}" + assert False, ( + f"\nThe parser was not completed," + f"\n file: {a}" + f"\n exception:\n {e.message}" + ) print("Parsing completed...\n") @@ -87,9 +93,11 @@ def test_files_from_TextIOWrapper_cli_with_generating_files(self): try: _ = self.parser.parsing(content=rdf_data, out=True) except Exception as e: - assert False, f"\nThe parser was not completed," \ - f"\n file: {a}" \ - f"\n exception:\n {e.message}" + assert False, ( + f"\nThe parser was not completed," + f"\n file: {a}" + f"\n exception:\n {e.message}" + ) print("Parsing completed...\n") @@ -109,9 +117,11 @@ def test_file_from_TextIOWrapper_cli_only_printing_result(self): try: _ = self.parser.parsing(content=rdf_data, out=False) except Exception as e: - assert False, f"\nThe parser was not completed," \ - f"\n file: {a}" \ - f"\n exception:\n {e.message}" + assert False, ( + f"\nThe parser was not completed," + f"\n file: {a}" + f"\n exception:\n {e.message}" + ) print("Parsing completed...\n") diff --git a/tests/sdmxattributes/test_observationStatus.py b/tests/sdmxattributes/test_observationStatus.py index b7f074d..0e027bf 100644 --- a/tests/sdmxattributes/test_observationStatus.py +++ b/tests/sdmxattributes/test_observationStatus.py @@ -29,43 +29,53 @@ def setUp(self): self.conversion = ObsStatus() def test_fix_value_data_in_predefined_values(self): - value = 'A' - expected = 'A' + value = "A" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nobsStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nobsStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) - value = 'a' - expected = 'A' + value = "a" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nobsStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nobsStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_fix_value_data_in_predefined_values_with_prefix(self): - value = 'obsStatus-A' - expected = 'A' + value = "obsStatus-A" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nobsStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nobsStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) - value = 'obsstatus-a' - expected = 'A' + value = "obsstatus-a" + expected = "A" obtained = self.conversion.fix_value(value=value) - assert obtained == expected, f"\nobsStatus was not the expected," \ - f"\n got : {obtained}" \ - f"\n expected: {expected}" + assert obtained == expected, ( + f"\nobsStatus was not the expected," + f"\n got : {obtained}" + f"\n expected: {expected}" + ) def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): - value = 'obsStatus-EEEEE' - expected = "obsStatus-EEEEE -> ObsStatus value is not included in the list of available values,\n" + \ - " got:obsStatus-EEEEE\n" + \ - " expected:['obsStatus-A', 'obsStatus-B', 'obsStatus-D', 'obsStatus-E', 'obsStatus-F', " \ - "'obsStatus-G', 'obsStatus-I', 'obsStatus-K', 'obsStatus-W', 'obsStatus-O', 'obsStatus-M', " \ - "'obsStatus-P', 'obsStatus-S', 'obsStatus-L', 'obsStatus-H', 'obsStatus-Q', 'obsStatus-J', " \ - "'obsStatus-N', 'obsStatus-U', 'obsStatus-V']" + value = "obsStatus-EEEEE" + expected = ( + "obsStatus-EEEEE -> ObsStatus value is not included in the list of available values,\n" + + " got:obsStatus-EEEEE\n" + + " expected:['obsStatus-A', 'obsStatus-B', 'obsStatus-D', 'obsStatus-E', 'obsStatus-F', " + "'obsStatus-G', 'obsStatus-I', 'obsStatus-K', 'obsStatus-W', 'obsStatus-O', 'obsStatus-M', " + "'obsStatus-P', 'obsStatus-S', 'obsStatus-L', 'obsStatus-H', 'obsStatus-Q', 'obsStatus-J', " + "'obsStatus-N', 'obsStatus-U', 'obsStatus-V']" + ) with self.assertRaises(ClassObsStatusError) as error: _ = self.conversion.fix_value(value=value) @@ -73,16 +83,16 @@ def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): self.assertEqual(str(error.exception), expected) def test_fix_value_unexpected_value_without_prefix(self): - value = 'EEEE' - expected = 'EEEE -> ObsStatus value is not the expected' + value = "EEEE" + expected = "EEEE -> ObsStatus value is not the expected" with self.assertRaises(ClassObsStatusError) as error: _ = self.conversion.fix_value(value=value) self.assertEqual(str(error.exception), expected) def test_fix_value_unexpected_prefix(self): - value = 'lkdjlks-A' - expected = 'lkdjlks-A -> ObsStatus value is not the expected' + value = "lkdjlks-A" + expected = "lkdjlks-A -> ObsStatus value is not the expected" with self.assertRaises(ClassObsStatusError) as error: _ = self.conversion.fix_value(value=value) diff --git a/tox.ini b/tox.ini index 457561e..cb476b6 100755 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,38 @@ [tox] -envlist = py37, py38, py39, py310, py311 +requires = + tox>=4 +env_list = lint, type, py{310, 311} -[testenv] +[testenv:py310] +description = run unit tests in python3.10 +deps = + pytest>=7 + pytest-sugar commands = - pytest tests/ \ No newline at end of file + pytest {posargs:tests} + +[testenv:py311] +description = run unit tests in python3.11 +deps = + pytest>=7 + pytest-sugar +commands = + pytest {posargs:tests} + +[testenv:lint] +description = run linters +skip_install = true +deps = + black==22.12 +commands = black {posargs:.} + +[testenv:type] +description = run type checks +deps = + mypy>=0.991 + types-docopt==0.6.11.3 + types-python-dateutil==2.8.19.14 + types-pytz==2023.3.0.0 + types-requests==2.31.0.2 +commands = + mypy {posargs:.} --no-warn-unused-ignores From ac905981afe590fdea8ae9926ed161e3de5803b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 25 Jul 2023 09:10:13 +0200 Subject: [PATCH 72/74] Update files with line-length to 120 (linter) --- .dockerignore | 2 +- api/custom_logging.py | 8 +-- api/server.py | 48 ++++------------ cli/command.py | 4 +- {doc => docs}/API.md | 0 ...RE IoTAgent-Turtle.postman_collection.json | 0 {doc => docs}/SDMX to JSON-LD.drawio | 0 {doc => docs}/api.yaml | 0 pyproject.toml | 47 ++++++++++++++++ sdmx2jsonld/common/datatypeconversion.py | 6 +- sdmx2jsonld/common/listmanagement.py | 12 +--- sdmx2jsonld/exceptions/__init__.py | 4 +- sdmx2jsonld/sdmxattributes/code.py | 4 +- sdmx2jsonld/sdmxattributes/dataComp.py | 3 +- sdmx2jsonld/sdmxattributes/sdmxattribute.py | 4 +- sdmx2jsonld/sdmxconcepts/confstatusconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/datacompconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/decimals.py | 3 +- sdmx2jsonld/sdmxconcepts/dissorgconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/freqconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/obsstatusconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/refareaconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/timeformatconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/timeperiodconcept.py | 3 +- sdmx2jsonld/sdmxconcepts/titleConcept.py | 3 +- sdmx2jsonld/sdmxconcepts/unitmultconcept.py | 3 +- sdmx2jsonld/sdmxdimensions/refarea.py | 3 +- sdmx2jsonld/transform/catalogue.py | 18 ++---- sdmx2jsonld/transform/concept.py | 13 +---- sdmx2jsonld/transform/conceptschema.py | 19 ++----- sdmx2jsonld/transform/context.py | 8 +-- sdmx2jsonld/transform/dataset.py | 52 ++++------------- sdmx2jsonld/transform/distribution.py | 8 +-- sdmx2jsonld/transform/entitytype.py | 13 +---- sdmx2jsonld/transform/observation.py | 56 +++++-------------- sdmx2jsonld/transform/property.py | 22 ++------ tests/common/test_classprecedence.py | 40 ++++--------- tests/common/test_commonclass.py | 12 +--- tests/common/test_datatypeconversion.py | 8 +-- tests/common/test_listmanagement.py | 4 +- tests/sdmxattributes/test_code.py | 24 ++------ .../sdmxattributes/test_confirmationStatus.py | 16 ++---- tests/sdmxattributes/test_examples.py | 18 +----- .../sdmxattributes/test_observationStatus.py | 16 ++---- 44 files changed, 165 insertions(+), 360 deletions(-) rename {doc => docs}/API.md (100%) rename {doc => docs}/FIWARE IoTAgent-Turtle.postman_collection.json (100%) rename {doc => docs}/SDMX to JSON-LD.drawio (100%) rename {doc => docs}/api.yaml (100%) diff --git a/.dockerignore b/.dockerignore index 3cd9d5e..a364d12 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ .venv/ -doc/ +docs/ examples/ test/ *.jsonld diff --git a/api/custom_logging.py b/api/custom_logging.py index 289f779..26c9712 100644 --- a/api/custom_logging.py +++ b/api/custom_logging.py @@ -70,15 +70,11 @@ def make_logger(cls, config_path: Path): return logger @classmethod - def customize_logging( - cls, filepath: Path, level: str, rotation: str, retention: str, format: str - ): + def customize_logging(cls, filepath: Path, level: str, rotation: str, retention: str, format: str): logger.remove() - logger.add( - sys.stdout, enqueue=True, backtrace=True, level=level.upper(), format=format - ) + logger.add(sys.stdout, enqueue=True, backtrace=True, level=level.upper(), format=format) logger.add( str(filepath), diff --git a/api/server.py b/api/server.py index f02cbaa..a3c339d 100644 --- a/api/server.py +++ b/api/server.py @@ -120,19 +120,13 @@ async def parse(request: Request, file: UploadFile, response: Response): if splitext(file.filename)[1] != ".ttl": # type: ignore[type-var] resp = {"message": "Allowed file type is only ttl"} response.status_code = status.HTTP_400_BAD_REQUEST - request.app.logger.error( - f'POST /parse 400 Bad Request, file: "{file.filename}"' - ) + request.app.logger.error(f'POST /parse 400 Bad Request, file: "{file.filename}"') else: try: content = await file.read() except Exception as e: - request.app.logger.error( - f'POST /parse 500 Problem reading file: "{file.filename}"' - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) + request.app.logger.error(f'POST /parse 500 Problem reading file: "{file.filename}"') + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) else: request.app.logger.info("File successfully read") @@ -146,26 +140,16 @@ async def parse(request: Request, file: UploadFile, response: Response): json_object = my_parser.parsing(content=StringIO(content)) # type: ignore[arg-type] except UnexpectedToken as e: request.app.logger.error(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) except UnexpectedInput as e: request.app.logger.error(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) except UnexpectedEOF as e: request.app.logger.error(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) except Exception as e: - request.app.logger.error( - f'POST /parse 500 Problem parsing file: "{file.filename}"' - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) + request.app.logger.error(f'POST /parse 500 Problem parsing file: "{file.filename}"') + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) else: request.app.logger.info(f"File successfully parsed") @@ -185,15 +169,11 @@ async def parse(request: Request, file: UploadFile, response: Response): # response.status_code = r.status_code except exceptions.Timeout as err: request.app.logger.error("Timeout requesting FIWARE Context Broker") - raise HTTPException( - status_code=status.HTTP_408_REQUEST_TIMEOUT, detail=str(err) - ) + raise HTTPException(status_code=status.HTTP_408_REQUEST_TIMEOUT, detail=str(err)) except exceptions.ConnectionError as err: message = f"There was a problem connecting to the FIWARE Context Broker. URL: {url}" request.app.logger.error(message) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(err) - ) + raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(err)) except exceptions.HTTPError as e: request.app.logger.error(f"Call to FIWARE Context Broker failed: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @@ -203,14 +183,10 @@ async def parse(request: Request, file: UploadFile, response: Response): except Exception as e: r = getattr(e, "message", str(e)) request.app.logger.error(r) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(r) - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(r)) else: request.app.logger.info(f"Content sent to the Context Broker") - request.app.logger.debug( - f"Status Code: {response.status_code}, Response:\n{resp}" - ) + request.app.logger.debug(f"Status Code: {response.status_code}, Response:\n{resp}") return resp diff --git a/cli/command.py b/cli/command.py index 191064e..17e3d39 100644 --- a/cli/command.py +++ b/cli/command.py @@ -65,9 +65,7 @@ def parse_cli() -> dict: schema = Schema( { "--help": bool, - "--input": Or( - None, Use(open, error="--input FILE, FILE should be readable") - ), + "--input": Or(None, Use(open, error="--input FILE, FILE should be readable")), "--output": bool, "--port": Or( None, diff --git a/doc/API.md b/docs/API.md similarity index 100% rename from doc/API.md rename to docs/API.md diff --git a/doc/FIWARE IoTAgent-Turtle.postman_collection.json b/docs/FIWARE IoTAgent-Turtle.postman_collection.json similarity index 100% rename from doc/FIWARE IoTAgent-Turtle.postman_collection.json rename to docs/FIWARE IoTAgent-Turtle.postman_collection.json diff --git a/doc/SDMX to JSON-LD.drawio b/docs/SDMX to JSON-LD.drawio similarity index 100% rename from doc/SDMX to JSON-LD.drawio rename to docs/SDMX to JSON-LD.drawio diff --git a/doc/api.yaml b/docs/api.yaml similarity index 100% rename from doc/api.yaml rename to docs/api.yaml diff --git a/pyproject.toml b/pyproject.toml index 09af394..325aaf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ requests = "2.31.0" rdflib = "6.3.2" python-dateutil = "2.8.2" +[tool.black] +line-length = 120 + [tool.poetry.dev-dependencies] pytest = "7.4.0" tox = "4.6.4" @@ -46,3 +49,47 @@ build-backend = "poetry.core.masonry.api" files = ["cli/command.py", "sdmx2jsonld/transform/parser.py"] search = '__version__ = "{current_version}"' replace = '__version__ = "{new_version}"' + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--tb=auto -ra --showlocals" + +[tool.coverage] +html.show_contexts = true +html.skip_covered = false +paths.source = [ + "api", + "cli", + "common", + "ngsild", + "sdmx2jsonld", + "tests", +] +report.fail_under = 88 +run.parallel = true +run.plugins = ["covdefaults"] + +[tool.mypy] +python_version = "3.11" +show_error_codes = true + +[[tool.mypy.overrides]] +module = [ + "api", + "cli", + "common", + "ngsild", + "sdmx2jsonld", + "tests", +] +ignore_missing_imports = true +warn_unused_ignores = true + +[tool.towncrier] +name = "tox" +filename = "docs/changelog.rst" +directory = "docs/changelog" +title_format = false +issue_format = ":issue:`{issue}`" +template = "docs/changelog/template.jinja2" +# possible types, all default: feature, bugfix, doc, removal, misc diff --git a/sdmx2jsonld/common/datatypeconversion.py b/sdmx2jsonld/common/datatypeconversion.py index 1a75e70..f1ad7a7 100644 --- a/sdmx2jsonld/common/datatypeconversion.py +++ b/sdmx2jsonld/common/datatypeconversion.py @@ -72,11 +72,7 @@ def stodt(value): raise Exception(f"Invalid format received: {type(value)}") result = self.correct_datatype_format(result) - result = ( - datetime.strptime(value, result) - .replace(tzinfo=timezone.utc) - .isoformat() - ) + result = datetime.strptime(value, result).replace(tzinfo=timezone.utc).isoformat() return result diff --git a/sdmx2jsonld/common/listmanagement.py b/sdmx2jsonld/common/listmanagement.py index c36dd7d..b07a0f1 100644 --- a/sdmx2jsonld/common/listmanagement.py +++ b/sdmx2jsonld/common/listmanagement.py @@ -42,9 +42,7 @@ def filter_key_with_prefix(prefix_key, not_allowed_keys, further_process_keys): logger.warning(f"The property {aux[1]} is not supported in statDCAT-AP") else: # These are the identified keys managed in a different way - logger.info( - f"The property {aux[1]} is manage afterwards in Dataset Class or in Property Class" - ) + logger.info(f"The property {aux[1]} is manage afterwards in Dataset Class or in Property Class") return False else: @@ -101,9 +99,7 @@ def get_rest_data(data, not_allowed_keys=None, further_process_keys=None): def extract_prefix(attribute): result = None if attribute is None or len(attribute) == 0: - raise ClassExtractPrefixError( - data=attribute, message=f"Unexpected data received: '{attribute}'" - ) + raise ClassExtractPrefixError(data=attribute, message=f"Unexpected data received: '{attribute}'") else: data = attribute.split(":") @@ -112,9 +108,7 @@ def extract_prefix(attribute): elif len(data) == 2: result = data[1] else: - raise ClassExtractPrefixError( - data=attribute, message=f"Unexpected number of prefixes: '{attribute}'" - ) + raise ClassExtractPrefixError(data=attribute, message=f"Unexpected number of prefixes: '{attribute}'") return result diff --git a/sdmx2jsonld/exceptions/__init__.py b/sdmx2jsonld/exceptions/__init__.py index dd06aa0..8feabd6 100644 --- a/sdmx2jsonld/exceptions/__init__.py +++ b/sdmx2jsonld/exceptions/__init__.py @@ -26,9 +26,7 @@ class UnexpectedEOF(LarkUnexpectedEOF): def __init__(self, expected, state=None, terminals_by_name=None): - super(LarkUnexpectedEOF, self).__init__( - expected=expected, state=state, terminals_by_name=terminals_by_name - ) + super(LarkUnexpectedEOF, self).__init__(expected=expected, state=state, terminals_by_name=terminals_by_name) class UnexpectedInput(LarkUnexpectedInput): diff --git a/sdmx2jsonld/sdmxattributes/code.py b/sdmx2jsonld/sdmxattributes/code.py index 3e55133..1b340ea 100644 --- a/sdmx2jsonld/sdmxattributes/code.py +++ b/sdmx2jsonld/sdmxattributes/code.py @@ -63,9 +63,7 @@ def fix_value(self, value): try: number = int(value) except ValueError: - raise ClassCode( - data=value, message=f"Data is not a valid value" - ) + raise ClassCode(data=value, message=f"Data is not a valid value") if number not in self.data_range: raise ClassCode( diff --git a/sdmx2jsonld/sdmxattributes/dataComp.py b/sdmx2jsonld/sdmxattributes/dataComp.py index 58c3fe4..37fa698 100644 --- a/sdmx2jsonld/sdmxattributes/dataComp.py +++ b/sdmx2jsonld/sdmxattributes/dataComp.py @@ -33,8 +33,7 @@ def __init__(self): super().__init__( entity_id="dataComp", label="Data Compilation", - description="Operations performed on data to derive new information according to a " - "given set of rules.", + description="Operations performed on data to derive new information according to a " "given set of rules.", concept_id="dataComp", identifier="dataComp", entity_range="xsd:string", diff --git a/sdmx2jsonld/sdmxattributes/sdmxattribute.py b/sdmx2jsonld/sdmxattributes/sdmxattribute.py index 4a33f8b..e4567bf 100644 --- a/sdmx2jsonld/sdmxattributes/sdmxattribute.py +++ b/sdmx2jsonld/sdmxattributes/sdmxattribute.py @@ -23,9 +23,7 @@ class SDMXAttribute(CommonClass): - def __init__( - self, entity_id, label, description, concept_id, identifier, entity_range - ): + def __init__(self, entity_id, label, description, concept_id, identifier, entity_range): super().__init__(entity="AttributeProperty") self.data = { "id": f"urn:ngsi-ld:AttributeProperty:{entity_id}", diff --git a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py index f9e1f5a..3d3254c 100644 --- a/sdmx2jsonld/sdmxconcepts/confstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/confstatusconcept.py @@ -36,6 +36,5 @@ def __init__(self): super().__init__( entity_id="confStatus", label="Confidentiality - status", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].CONF_STATUS", ) diff --git a/sdmx2jsonld/sdmxconcepts/datacompconcept.py b/sdmx2jsonld/sdmxconcepts/datacompconcept.py index 6b97de9..71f0bd1 100644 --- a/sdmx2jsonld/sdmxconcepts/datacompconcept.py +++ b/sdmx2jsonld/sdmxconcepts/datacompconcept.py @@ -35,6 +35,5 @@ def __init__(self): super().__init__( entity_id="dataComp", label="Data Compilation", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=" - "SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=" "SDMX:CROSS_DOMAIN_CONCEPTS[1.0].DATA_COMP", ) diff --git a/sdmx2jsonld/sdmxconcepts/decimals.py b/sdmx2jsonld/sdmxconcepts/decimals.py index 6ce8866..c0eec8c 100644 --- a/sdmx2jsonld/sdmxconcepts/decimals.py +++ b/sdmx2jsonld/sdmxconcepts/decimals.py @@ -34,6 +34,5 @@ def __init__(self): super().__init__( entity_id="decimals", label="Decimals", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].DECIMALS", ) diff --git a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py index e3a4b8f..53bd28b 100644 --- a/sdmx2jsonld/sdmxconcepts/dissorgconcept.py +++ b/sdmx2jsonld/sdmxconcepts/dissorgconcept.py @@ -34,6 +34,5 @@ def __init__(self): super().__init__( entity_id="dissOrg", label="Data Dissemination Agency", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].DISS_ORG", ) diff --git a/sdmx2jsonld/sdmxconcepts/freqconcept.py b/sdmx2jsonld/sdmxconcepts/freqconcept.py index 8aa2a0b..59a071c 100644 --- a/sdmx2jsonld/sdmxconcepts/freqconcept.py +++ b/sdmx2jsonld/sdmxconcepts/freqconcept.py @@ -33,6 +33,5 @@ def __init__(self): super().__init__( entity_id="freq", label="Frequency", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].FREQ", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].FREQ", ) diff --git a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py index 6a6aeb4..8c753ca 100644 --- a/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py +++ b/sdmx2jsonld/sdmxconcepts/obsstatusconcept.py @@ -34,6 +34,5 @@ def __init__(self): super().__init__( entity_id="obsStatus", label="Observation Status", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].OBS_STATUS", ) diff --git a/sdmx2jsonld/sdmxconcepts/refareaconcept.py b/sdmx2jsonld/sdmxconcepts/refareaconcept.py index 232b163..2a16c2f 100644 --- a/sdmx2jsonld/sdmxconcepts/refareaconcept.py +++ b/sdmx2jsonld/sdmxconcepts/refareaconcept.py @@ -33,6 +33,5 @@ def __init__(self): super().__init__( entity_id="refArea", label="Reference Area", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].REF_AREA", ) diff --git a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py index 85c7e30..72d2f08 100644 --- a/sdmx2jsonld/sdmxconcepts/timeformatconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeformatconcept.py @@ -34,6 +34,5 @@ def __init__(self): super().__init__( entity_id="timeFormat", label="Time Format", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].TIME_FORMAT", ) diff --git a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py index 6e5acc0..893f571 100644 --- a/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py +++ b/sdmx2jsonld/sdmxconcepts/timeperiodconcept.py @@ -34,6 +34,5 @@ def __init__(self): super().__init__( entity_id="timePeriod", label="Time Period", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].TIME_PERIOD", ) diff --git a/sdmx2jsonld/sdmxconcepts/titleConcept.py b/sdmx2jsonld/sdmxconcepts/titleConcept.py index 539e99e..43fdedf 100644 --- a/sdmx2jsonld/sdmxconcepts/titleConcept.py +++ b/sdmx2jsonld/sdmxconcepts/titleConcept.py @@ -33,6 +33,5 @@ def __init__(self): super().__init__( entity_id="title", label="Title", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].TITLE", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].TITLE", ) diff --git a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py index b01f8d4..e53cd3d 100644 --- a/sdmx2jsonld/sdmxconcepts/unitmultconcept.py +++ b/sdmx2jsonld/sdmxconcepts/unitmultconcept.py @@ -35,6 +35,5 @@ def __init__(self): super().__init__( entity_id="unitMult", label="Unit Multiplier", - notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" - "CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT", + notation="urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=SDMX:" "CROSS_DOMAIN_CONCEPTS[1.0].UNIT_MULT", ) diff --git a/sdmx2jsonld/sdmxdimensions/refarea.py b/sdmx2jsonld/sdmxdimensions/refarea.py index f907570..d802c18 100644 --- a/sdmx2jsonld/sdmxdimensions/refarea.py +++ b/sdmx2jsonld/sdmxdimensions/refarea.py @@ -33,8 +33,7 @@ def __init__(self): super().__init__( entity_id="refArea", label="Reference Area", - description="The country or geographic area to which the measured statistical " - "phenomenon relates.", + description="The country or geographic area to which the measured statistical " "phenomenon relates.", concept_id="refArea", identifier="refArea", entity_range="xsd:string", diff --git a/sdmx2jsonld/transform/catalogue.py b/sdmx2jsonld/transform/catalogue.py index 7e86123..92ef1a7 100644 --- a/sdmx2jsonld/transform/catalogue.py +++ b/sdmx2jsonld/transform/catalogue.py @@ -91,9 +91,7 @@ def add_data(self, title, dataset_id, data): index, key, value = get_property_value(data=data, property_name="issued") if index != -1: # We found an 'issued' data - self.data.update( - self.__generate_property__(key="releaseDate", value=value[0][0]) - ) + self.data.update(self.__generate_property__(key="releaseDate", value=value[0][0])) # Get the rest of the data, qb:structure has the same value of qb:dataset, so we decide to # use only qb:dataset in CatalogueDCAT-AP @@ -117,10 +115,7 @@ def patch_data(self, data, language_map): else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [ - self.data.update(self.__generate_property__(key=k, value=v)) - for k, v in data.items() - ] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] def __complete_label__(self, title, data): try: @@ -134,16 +129,11 @@ def __complete_label__(self, title, data): try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning( - f"The Catalogue {title} has a " - f"rdfs:label without language tag: {description}" - ) + logger.warning(f"The Catalogue {title} has a " f"rdfs:label without language tag: {description}") aux = len(description) if aux != 1: - logger.error( - f"Catalogue: there is more than 1 description ({aux}), values: {description}" - ) + logger.error(f"Catalogue: there is more than 1 description ({aux}), values: {description}") else: # There is no language tag, we use by default 'en' languages = ["en"] diff --git a/sdmx2jsonld/transform/concept.py b/sdmx2jsonld/transform/concept.py index 71fc1c0..3e7c54c 100644 --- a/sdmx2jsonld/transform/concept.py +++ b/sdmx2jsonld/transform/concept.py @@ -80,16 +80,11 @@ def add_data(self, concept_id, data): try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning( - f"The Concept {concept_id} has a " - f"skos:prefLabel without language tag: {description}" - ) + logger.warning(f"The Concept {concept_id} has a " f"skos:prefLabel without language tag: {description}") aux = len(description) if aux != 1: - logger.error( - f"Concept: there is more than 1 description ({aux}), values: {description}" - ) + logger.error(f"Concept: there is more than 1 description ({aux}), values: {description}") else: # There is no language tag, we use by default 'en' languages = ["en"] @@ -139,9 +134,7 @@ def need_add_subclass(self, data): position = data.index("rdfs:subClassOf") + 1 self.data["rdfs:subClassOf"]["value"] = data[position][0] except ValueError: - logger.info( - f"The Concept {self.concept_id} has no rdfs:subClassOf property, deleting the key in the data" - ) + logger.info(f"The Concept {self.concept_id} has no rdfs:subClassOf property, deleting the key in the data") # We delete the "rdfs:subClassOf" property from the final structure self.data.pop("rdfs:subClassOf") diff --git a/sdmx2jsonld/transform/conceptschema.py b/sdmx2jsonld/transform/conceptschema.py index df21d04..6764411 100644 --- a/sdmx2jsonld/transform/conceptschema.py +++ b/sdmx2jsonld/transform/conceptschema.py @@ -67,15 +67,12 @@ def add_data(self, concept_schema_id, data): languages = [x[1].replace("@", "").lower() for x in description] except IndexError: logger.warning( - f"The ConceptSchema {concept_schema_id} has a " - f"skos:prefLabel without language tag: {description}" + f"The ConceptSchema {concept_schema_id} has a " f"skos:prefLabel without language tag: {description}" ) aux = len(description) if aux != 1: - logger.error( - f"ConceptSchema: there is more than 1 description ({aux}), values: {description}" - ) + logger.error(f"ConceptSchema: there is more than 1 description ({aux}), values: {description}") else: # There is no language tag, we use by default 'en' languages = ["en"] @@ -102,9 +99,7 @@ def add_data(self, concept_schema_id, data): # TODO: We need to control that the concept id extracted here are the same that we analyse afterwards. # skos:hasTopConcept, this is a list of ids position = data.index("skos:hasTopConcept") + 1 - result = list( - map(lambda x: self.generate_id(value=x, entity="Concept"), data[position]) - ) + result = list(map(lambda x: self.generate_id(value=x, entity="Concept"), data[position])) self.data["hasTopConcept"]["object"] = result # Get the rest of data, dct:created and dct:modified properties @@ -115,9 +110,7 @@ def add_data(self, concept_schema_id, data): "value": flatten_value(data[position]), } except ValueError: - logger.warning( - f"dct:created is not present in the Concept Schema: {concept_schema_id}" - ) + logger.warning(f"dct:created is not present in the Concept Schema: {concept_schema_id}") try: position = data.index("dct:modified") + 1 @@ -126,9 +119,7 @@ def add_data(self, concept_schema_id, data): "value": flatten_value(data[position]), } except ValueError: - logger.warning( - f"dct:modified is not present in the Concept Schema: {concept_schema_id}" - ) + logger.warning(f"dct:modified is not present in the Concept Schema: {concept_schema_id}") # Order the keys in the final json-ld a = Context() diff --git a/sdmx2jsonld/transform/context.py b/sdmx2jsonld/transform/context.py index c19e3e5..60d7ed7 100644 --- a/sdmx2jsonld/transform/context.py +++ b/sdmx2jsonld/transform/context.py @@ -38,9 +38,7 @@ def __init__(self): # By default, the context should include the smart data models context self.context["@context"].update( - { - "sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld" - } + {"sdmp": "https://smart-data-models.github.io/dataModel.STAT-DCAT-AP/context.jsonld"} ) # statDCAT-AP contexts @@ -48,9 +46,7 @@ def __init__(self): self.context["@context"].update({"dct": "http://purl.org/dc/terms/"}) - self.context["@context"].update( - {"stat": "http://data.europa.eu/(xyz)/statdcat-ap/"} - ) + self.context["@context"].update({"stat": "http://data.europa.eu/(xyz)/statdcat-ap/"}) def add_context(self, context): aux = list(context.items()) diff --git a/sdmx2jsonld/transform/dataset.py b/sdmx2jsonld/transform/dataset.py index 14f618a..3d87a6f 100644 --- a/sdmx2jsonld/transform/dataset.py +++ b/sdmx2jsonld/transform/dataset.py @@ -121,29 +121,15 @@ def __init__(self): "qb:measure": { "entity": "statUnitMeasure", "key": "statUnitMeasure", - "value": { - "statUnitMeasure": {"type": "Relationship", "object": list()} - }, + "value": {"statUnitMeasure": {"type": "Relationship", "object": list()}}, }, } self.keys = ( {k: k for k in self.data.keys()} - | { - self.components["qb:attribute"]["key"]: self.components["qb:attribute"][ - "key" - ] - } - | { - self.components["qb:dimension"]["key"]: self.components["qb:dimension"][ - "key" - ] - } - | { - self.components["qb:measure"]["key"]: self.components["qb:measure"][ - "key" - ] - } + | {self.components["qb:attribute"]["key"]: self.components["qb:attribute"]["key"]} + | {self.components["qb:dimension"]["key"]: self.components["qb:dimension"]["key"]} + | {self.components["qb:measure"]["key"]: self.components["qb:measure"]["key"]} ) self.sdmx_dimensions = { @@ -201,9 +187,7 @@ def add_components(self, component): position = component.index(type_component) + 1 if type_component == "qb:measure": - logger.info( - f'The qb:measure "{component[position][0]}" is not manage in statDCAT-AP' - ) + logger.info(f'The qb:measure "{component[position][0]}" is not manage in statDCAT-AP') new_component, new_concept, new_concept_schema = None, None, None else: new_component, new_concept, new_concept_schema = self.manage_components( @@ -216,16 +200,12 @@ def manage_components(self, type_component, component, position): new_component, new_concept, new_concept_schema = None, None, None try: entity = self.components[type_component]["entity"] - name, new_id = self.generate_id( - entity=entity, value=component[position][0], update_id=False - ) + name, new_id = self.generate_id(entity=entity, value=component[position][0], update_id=False) key = self.components[type_component]["key"] # It is possible that the original file contains already the description if new_id in self.components[type_component]["value"][key]["object"]: - logger.warning( - f"The component {new_id} is duplicated and already defined in the {self.data['id']}" - ) + logger.warning(f"The component {new_id} is duplicated and already defined in the {self.data['id']}") elif name in self.list_special_components: # We need to create manually the description of these dimensions, concepts, and conceptschemas logger.warning( @@ -242,9 +222,7 @@ def manage_components(self, type_component, component, position): self.components[type_component]["value"][key]["object"].append(new_id) self.data = self.data | self.components[type_component]["value"] except ValueError: - logger.error( - f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}" - ) + logger.error(f"Error, it was identified a qb:ComponentSpecification with a wrong type: {type_component}") # Order the keys in the final json-ld a = Context() @@ -291,10 +269,7 @@ def patch_data(self, data, language_map): else: # TODO: Add only those properties that are expected, if they are not know or unexpected discard and provide # a logging about the property is discarded due to it is not considered in the statSCAT-AP spec. - [ - self.data.update(self.__generate_property__(key=k, value=v)) - for k, v in data.items() - ] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] def __complete_label__(self, title, data): try: @@ -308,16 +283,11 @@ def __complete_label__(self, title, data): try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning( - f"The Dataset {title} has a " - f"rdfs:label without language tag: {description}" - ) + logger.warning(f"The Dataset {title} has a " f"rdfs:label without language tag: {description}") aux = len(description) if aux != 1: - logger.error( - f"Dataset: there is more than 1 description ({aux}), values: {description}" - ) + logger.error(f"Dataset: there is more than 1 description ({aux}), values: {description}") else: # There is no language tag, we use by default 'en' languages = ["en"] diff --git a/sdmx2jsonld/transform/distribution.py b/sdmx2jsonld/transform/distribution.py index 398f344..8908c11 100644 --- a/sdmx2jsonld/transform/distribution.py +++ b/sdmx2jsonld/transform/distribution.py @@ -36,9 +36,7 @@ def __init__(self): "type": "Distribution", "accessUrl": { "type": "Property", - "value": [ - "/ngsi-ld/v1/entities?type=https://smartdatamodels.org/dataModel.SDMX/Observation" - ], + "value": ["/ngsi-ld/v1/entities?type=https://smartdatamodels.org/dataModel.SDMX/Observation"], }, "description": { "type": "Property", @@ -71,6 +69,4 @@ def generate_data(self, catalogue): with open(config_path) as config_file: config = load(config_file) - self.data["accessUrl"]["value"][0] = ( - config["broker"] + self.data["accessUrl"]["value"][0] - ) + self.data["accessUrl"]["value"][0] = config["broker"] + self.data["accessUrl"]["value"][0] diff --git a/sdmx2jsonld/transform/entitytype.py b/sdmx2jsonld/transform/entitytype.py index 4ed1102..c790356 100644 --- a/sdmx2jsonld/transform/entitytype.py +++ b/sdmx2jsonld/transform/entitytype.py @@ -105,9 +105,7 @@ def __find_entity_type__(self, string): is_new = True except ValueError: - logger.info( - f"Not a definition triples {string}, need to find the proper structure" - ) + logger.info(f"Not a definition triples {string}, need to find the proper structure") is_new = False data = self.__get_subject__(title=string[0]) string1 = string[1:] @@ -120,9 +118,7 @@ def transform(self, string): if is_new: self.create_data(entity_type=data_type, data=new_string, title=string[0]) else: - logger.info( - f"Checking previous subjects to find if it was created previously" - ) + logger.info(f"Checking previous subjects to find if it was created previously") self.patch_data(datatype=data_type, data=new_string) def patch_data(self, datatype, data): @@ -137,10 +133,7 @@ def flatten_value(y): flatten_data = [item for sublist in data for item in sublist] if flatten_data[0] != "rdfs:label": - flatten_data = { - flatten_data[i]: flatten_value(flatten_data[i + 1]) - for i in range(0, len(flatten_data), 2) - } + flatten_data = {flatten_data[i]: flatten_value(flatten_data[i + 1]) for i in range(0, len(flatten_data), 2)} language_map = False else: language_map = True diff --git a/sdmx2jsonld/transform/observation.py b/sdmx2jsonld/transform/observation.py index eb8fd72..99493cf 100644 --- a/sdmx2jsonld/transform/observation.py +++ b/sdmx2jsonld/transform/observation.py @@ -51,9 +51,7 @@ def __init__(self): "timePeriod": {"type": "Property", "value": str()}, "obsValue": {"type": "Property", "value": float()}, "dimensions": {"type": "Property", "value": list()}, - "@context": [ - "https://raw.githubusercontent.com/smart-data-models/dataModel.SDMX/master/context.jsonld" - ], + "@context": ["https://raw.githubusercontent.com/smart-data-models/dataModel.SDMX/master/context.jsonld"], } self.concept_id = str() @@ -65,9 +63,7 @@ def add_data(self, title, observation_id, data): # and an observation # Add the confStatus - key = self.__assign_property__( - requested_key="sdmx-attribute:confStatus", data=data - ) + key = self.__assign_property__(requested_key="sdmx-attribute:confStatus", data=data) self.data[key]["value"] = ConfStatus().fix_value(value=self.data[key]["value"]) # Add the id @@ -78,26 +74,16 @@ def add_data(self, title, observation_id, data): self.data["title"]["value"] = title # Add the decimals - key = self.__assign_property__( - requested_key="sdmx-attribute:decimals", data=data - ) - self.data[key]["value"] = Code(typecode=key).fix_value( - value=self.data[key]["value"] - ) + key = self.__assign_property__(requested_key="sdmx-attribute:decimals", data=data) + self.data[key]["value"] = Code(typecode=key).fix_value(value=self.data[key]["value"]) # Add obsStatus - key = self.__assign_property__( - requested_key="sdmx-attribute:obsStatus", data=data - ) + key = self.__assign_property__(requested_key="sdmx-attribute:obsStatus", data=data) self.data[key]["value"] = ObsStatus().fix_value(value=self.data[key]["value"]) # Add unitMult - key = self.__assign_property__( - requested_key="sdmx-attribute:unitMult", data=data - ) - self.data[key]["value"] = Code(typecode=key).fix_value( - value=self.data[key]["value"] - ) + key = self.__assign_property__(requested_key="sdmx-attribute:unitMult", data=data) + self.data[key]["value"] = Code(typecode=key).fix_value(value=self.data[key]["value"]) # Add freq "pattern": "^_[OUZ]|[SQBNI]|OA|OM|[AMWDH]_*[0-9]*$" # TODO: Add verification of coded data following the pattern @@ -109,18 +95,14 @@ def add_data(self, title, observation_id, data): _ = self.__assign_property__(requested_key="sdmx-dimension:refArea", data=data) # Add timePeriod - _ = self.__assign_property__( - requested_key="sdmx-dimension:timePeriod", data=data - ) + _ = self.__assign_property__(requested_key="sdmx-dimension:timePeriod", data=data) # Add obsValue _ = self.__assign_property__(requested_key="sdmx-measure:obsValue", data=data) # Add dataset parser = RegParser() - key = self.__assign_property__( - requested_key="qb:dataSet", data=data, key_property="object" - ) + key = self.__assign_property__(requested_key="qb:dataSet", data=data, key_property="object") identifier = parser.obtain_id(self.data[key]["object"]) self.data[key]["object"] = "urn:ngsi-ld:CatalogueDCAT-AP:" + identifier @@ -137,26 +119,16 @@ def create_data(key, value): parser = RegParser() # We need to get the list of available dimension keys - dimension_keys = [ - a for a in data if self.regexpDimension.match(a.__str__()) is not None - ] + dimension_keys = [a for a in data if self.regexpDimension.match(a.__str__()) is not None] # We need to get the list of values for these keys, if there is an url, it is considered a values = [self.get_data(data[data.index(a) + 1]) for a in dimension_keys] - values = [ - parser.obtain_id(a, prefix_string="urn:ngsi-ld:Concept:") for a in values - ] + values = [parser.obtain_id(a, prefix_string="urn:ngsi-ld:Concept:") for a in values] # Get the list of Entity IDs - entity_ids = [ - "urn:ngsi-ld:DimensionProperty:" + b.split("ns1:", 1)[1] - for b in dimension_keys - ] - - result = [ - create_data(key=entity_ids[i], value=values[i]) - for i in range(0, len(values)) - ] + entity_ids = ["urn:ngsi-ld:DimensionProperty:" + b.split("ns1:", 1)[1] for b in dimension_keys] + + result = [create_data(key=entity_ids[i], value=values[i]) for i in range(0, len(values))] return result diff --git a/sdmx2jsonld/transform/property.py b/sdmx2jsonld/transform/property.py index c5907ce..e06c5dc 100644 --- a/sdmx2jsonld/transform/property.py +++ b/sdmx2jsonld/transform/property.py @@ -67,16 +67,11 @@ def add_data(self, property_id, data): try: languages = [x[1].replace("@", "").lower() for x in description] except IndexError: - logger.warning( - f"The Property {property_id} has a " - f"rdfs:label without language tag: {description}" - ) + logger.warning(f"The Property {property_id} has a " f"rdfs:label without language tag: {description}") aux = len(description) if aux != 1: - logger.error( - f"Property: there is more than 1 description ({aux}), values: {description}" - ) + logger.error(f"Property: there is more than 1 description ({aux}), values: {description}") else: # There is no language tag, we use by default 'en' languages = ["en"] @@ -100,14 +95,10 @@ def add_data(self, property_id, data): # TODO: We need to control that the codeList id extracted here are the same that we analyse afterwards. try: position = data.index("qb:codeList") + 1 - code_list = self.generate_id( - entity="ConceptSchema", value=data[position][0] - ) + code_list = self.generate_id(entity="ConceptSchema", value=data[position][0]) self.data["codeList"]["object"] = code_list except ValueError: - logger.warning( - f"Property: {property_id} has not qb:codeList, deleting the key in the data" - ) + logger.warning(f"Property: {property_id} has not qb:codeList, deleting the key in the data") # If we have not the property, we delete it from data self.data.pop("codeList") @@ -135,10 +126,7 @@ def add_data(self, property_id, data): ) # add the new data to the dataset structure - [ - self.data.update(self.__generate_property__(key=k, value=v)) - for k, v in data.items() - ] + [self.data.update(self.__generate_property__(key=k, value=v)) for k, v in data.items()] # Order the keys in the final json-ld a = Context() diff --git a/tests/common/test_classprecedence.py b/tests/common/test_classprecedence.py index f433139..fcaab52 100644 --- a/tests/common/test_classprecedence.py +++ b/tests/common/test_classprecedence.py @@ -37,9 +37,7 @@ def test_precedence_one_class(self): """ obtained = self.pre.precedence(["qb:DataStructureDefinition"]) expected = "qb:DataStructureDefinition" - assert ( - obtained == expected - ), f"'qb:DataStructureDefinition' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:DataStructureDefinition' expected, got: '{obtained}'" obtained = self.pre.precedence(["skos:Concept"]) expected = "skos:Concept" @@ -51,15 +49,11 @@ def test_precedence_one_class(self): obtained = self.pre.precedence(["qb:DimensionProperty"]) expected = "qb:DimensionProperty" - assert ( - obtained == expected - ), f"'qb:DimensionProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" obtained = self.pre.precedence(["qb:AttributeProperty"]) expected = "qb:AttributeProperty" - assert ( - obtained == expected - ), f"'qb:AttributeProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" obtained = self.pre.precedence(["skos:ConceptScheme"]) expected = "skos:ConceptScheme" @@ -71,9 +65,7 @@ def test_precedence_one_class(self): obtained = self.pre.precedence(["qb:ComponentSpecification"]) expected = "qb:ComponentSpecification" - assert ( - obtained == expected - ), f"'qb:ComponentSpecification' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:ComponentSpecification' expected, got: '{obtained}'" obtained = self.pre.precedence(["qb:MeasureProperty"]) expected = "qb:MeasureProperty" @@ -128,38 +120,26 @@ def test_precedence_classes_with_class_values(self): def test_attribute_and_coded_property(self): obtained = self.pre.precedence(["qb:AttributeProperty", "qb:CodedProperty"]) expected = "qb:AttributeProperty" - assert ( - obtained == expected - ), f"'qb:AttributeProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" obtained = self.pre.precedence(["qb:CodedProperty", "qb:AttributeProperty"]) expected = "qb:AttributeProperty" - assert ( - obtained == expected - ), f"'qb:AttributeProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:AttributeProperty' expected, got: '{obtained}'" def test_coded_and_dimension_property(self): obtained = self.pre.precedence(["qb:CodedProperty", "qb:DimensionProperty"]) expected = "qb:DimensionProperty" - assert ( - obtained == expected - ), f"'qb:DimensionProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" obtained = self.pre.precedence(["qb:DimensionProperty", "qb:CodedProperty"]) expected = "qb:DimensionProperty" - assert ( - obtained == expected - ), f"'qb:DimensionProperty' expected, got: '{obtained}'" + assert obtained == expected, f"'qb:DimensionProperty' expected, got: '{obtained}'" def test_concept_and_other_property(self): - obtained = self.pre.precedence( - ["skos:Concept", ""] - ) + obtained = self.pre.precedence(["skos:Concept", ""]) expected = "skos:Concept" assert obtained == expected, f"'skos:Concept' expected, got: '{obtained}'" - obtained = self.pre.precedence( - ["", "skos:Concept"] - ) + obtained = self.pre.precedence(["", "skos:Concept"]) expected = "skos:Concept" assert obtained == expected, f"'skos:Concept' expected, got: '{obtained}'" diff --git a/tests/common/test_commonclass.py b/tests/common/test_commonclass.py index 0896b9b..e5d9479 100644 --- a/tests/common/test_commonclass.py +++ b/tests/common/test_commonclass.py @@ -32,9 +32,7 @@ def setUp(self) -> None: def test_instance_class(self): cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id( - "https://string-to-parse-ur/entity_id", update_id=True - ) + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) assert urnid == "urn:ngsi-ld:test.common.entity:entity_id" # urnid = cclass.generate_id("") # print(urnid) @@ -49,9 +47,7 @@ def test_save(self): "status": "ngsi-ld:status", } cclass = CommonClass("test.common.entity") - urnid = cclass.generate_id( - "https://string-to-parse-ur/entity_id", update_id=True - ) + urnid = cclass.generate_id("https://string-to-parse-ur/entity_id", update_id=True) assert urnid == "urn:ngsi-ld:test.common.entity:entity_id" cclass.add_context(context, context_map) @@ -60,9 +56,7 @@ def test_save(self): os.chdir("/tmp/commonclass") cclass.save() - with open( - "/tmp/commonclass/output/test.common.entity_entity_id.jsonld", "r" - ) as f: + with open("/tmp/commonclass/output/test.common.entity_entity_id.jsonld", "r") as f: data = json.load(f) assert data["id"] == urnid assert data["@context"] == context["@context"] diff --git a/tests/common/test_datatypeconversion.py b/tests/common/test_datatypeconversion.py index 5726d54..b3b2340 100755 --- a/tests/common/test_datatypeconversion.py +++ b/tests/common/test_datatypeconversion.py @@ -127,14 +127,10 @@ def test_datetime_string_conversion_1(self): """ Check if we can get a correct datetime value from a string, case 1 """ - obtained = self.conversion.convert( - '"2022-01-15T08:00:00.000 UTC"', "xsd:dateTime" - ) + obtained = self.conversion.convert('"2022-01-15T08:00:00.000 UTC"', "xsd:dateTime") expected = "2022-01-15T08:00:00+00:00" assert obtained == expected, ( - f"\n\nDateTime was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\n\nDateTime was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_int_string_conversion(self): diff --git a/tests/common/test_listmanagement.py b/tests/common/test_listmanagement.py index 2d5d003..58f7e10 100644 --- a/tests/common/test_listmanagement.py +++ b/tests/common/test_listmanagement.py @@ -190,9 +190,7 @@ def test_get_property_data_from_array_property_with_prefix(self): ['"PIB et principales composantes (prix courants)"', "@fr"], ] - index, key, obtained = get_property_value( - data=data, property_name="dcterms:title" - ) + index, key, obtained = get_property_value(data=data, property_name="dcterms:title") self.assertEqual(index, 8) self.assertEqual(key, "dcterms:title") diff --git a/tests/sdmxattributes/test_code.py b/tests/sdmxattributes/test_code.py index 83201e7..f79db6d 100644 --- a/tests/sdmxattributes/test_code.py +++ b/tests/sdmxattributes/test_code.py @@ -32,16 +32,12 @@ def test_code_value_with_prefix(self): code = Code(typecode="decimals") obtained = code.fix_value(value=value) assert obtained == expected, ( - f"\ncode was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\ncode was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_code_negative_value_with_prefix(self): value = "sdmx-code:decimals--1" - expected = ( - "sdmx-code:decimals--1 -> decimals out of range, got: -1 range(0, 15)" - ) + expected = "sdmx-code:decimals--1 -> decimals out of range, got: -1 range(0, 15)" code = Code(typecode="decimals") @@ -52,9 +48,7 @@ def test_code_negative_value_with_prefix(self): def test_code_value_bigger_than_maximum_with_prefix(self): value = "sdmx-code:decimals-67" - expected = ( - "sdmx-code:decimals-67 -> decimals out of range, got: 67 range(0, 15)" - ) + expected = "sdmx-code:decimals-67 -> decimals out of range, got: 67 range(0, 15)" code = Code(typecode="decimals") @@ -70,9 +64,7 @@ def test_code_value_without_prefix(self): code = Code(typecode="unitMult") obtained = code.fix_value(value=value) assert obtained == expected, ( - f"\ncode was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\ncode was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_code_negative_value_without_prefix(self): @@ -104,9 +96,7 @@ def test_code_integer_value(self): code = Code(typecode="unitMult") obtained = code.fix_value(value=value) assert obtained == expected, ( - f"\ncode was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\ncode was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_code_integer_value_out_of_range(self): @@ -127,9 +117,7 @@ def test_code_string_value(self): code = Code(typecode="unitMult") obtained = code.fix_value(value=value) assert obtained == expected, ( - f"\ncode was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\ncode was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_code_string_value_out_of_range(self): diff --git a/tests/sdmxattributes/test_confirmationStatus.py b/tests/sdmxattributes/test_confirmationStatus.py index fc24c86..b5406b4 100644 --- a/tests/sdmxattributes/test_confirmationStatus.py +++ b/tests/sdmxattributes/test_confirmationStatus.py @@ -33,18 +33,14 @@ def test_fix_value_data_in_predefined_values(self): expected = "F" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nconfStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nconfStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) value = "f" expected = "F" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nconfStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nconfStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_fix_value_data_in_predefined_values_with_prefix(self): @@ -52,18 +48,14 @@ def test_fix_value_data_in_predefined_values_with_prefix(self): expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nconfStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nconfStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) value = "confstatus-a" expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nconfStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nconfStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): diff --git a/tests/sdmxattributes/test_examples.py b/tests/sdmxattributes/test_examples.py index 2639354..dcf0c29 100644 --- a/tests/sdmxattributes/test_examples.py +++ b/tests/sdmxattributes/test_examples.py @@ -69,11 +69,7 @@ def test_files_from_StringIO_web_interface_loop(self): try: _ = self.parser.parsing(content=StringIO(rdf_data), out=False) except Exception as e: - assert False, ( - f"\nThe parser was not completed," - f"\n file: {a}" - f"\n exception:\n {e.message}" - ) + assert False, f"\nThe parser was not completed," f"\n file: {a}" f"\n exception:\n {e.message}" print("Parsing completed...\n") @@ -93,11 +89,7 @@ def test_files_from_TextIOWrapper_cli_with_generating_files(self): try: _ = self.parser.parsing(content=rdf_data, out=True) except Exception as e: - assert False, ( - f"\nThe parser was not completed," - f"\n file: {a}" - f"\n exception:\n {e.message}" - ) + assert False, f"\nThe parser was not completed," f"\n file: {a}" f"\n exception:\n {e.message}" print("Parsing completed...\n") @@ -117,11 +109,7 @@ def test_file_from_TextIOWrapper_cli_only_printing_result(self): try: _ = self.parser.parsing(content=rdf_data, out=False) except Exception as e: - assert False, ( - f"\nThe parser was not completed," - f"\n file: {a}" - f"\n exception:\n {e.message}" - ) + assert False, f"\nThe parser was not completed," f"\n file: {a}" f"\n exception:\n {e.message}" print("Parsing completed...\n") diff --git a/tests/sdmxattributes/test_observationStatus.py b/tests/sdmxattributes/test_observationStatus.py index 0e027bf..48195e9 100644 --- a/tests/sdmxattributes/test_observationStatus.py +++ b/tests/sdmxattributes/test_observationStatus.py @@ -33,18 +33,14 @@ def test_fix_value_data_in_predefined_values(self): expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nobsStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nobsStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) value = "a" expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nobsStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nobsStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_fix_value_data_in_predefined_values_with_prefix(self): @@ -52,18 +48,14 @@ def test_fix_value_data_in_predefined_values_with_prefix(self): expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nobsStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nobsStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) value = "obsstatus-a" expected = "A" obtained = self.conversion.fix_value(value=value) assert obtained == expected, ( - f"\nobsStatus was not the expected," - f"\n got : {obtained}" - f"\n expected: {expected}" + f"\nobsStatus was not the expected," f"\n got : {obtained}" f"\n expected: {expected}" ) def test_fix_value_data_with_valid_prefix_but_not_expected_value(self): From 05ba69f1359bdd2723430b42130095d313a4c6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 25 Jul 2023 12:09:37 +0200 Subject: [PATCH 73/74] Final configuration of the poetry and tox --- .coveragerc | 39 ++++++++++ .dockerignore | 1 - poetry.lock | 75 ++++++++++++++++++- pyproject.toml | 12 +-- requirements-dev.txt => test-requirements.txt | 1 + tox.ini | 14 +++- 6 files changed, 128 insertions(+), 14 deletions(-) create mode 100755 .coveragerc rename requirements-dev.txt => test-requirements.txt (93%) diff --git a/.coveragerc b/.coveragerc new file mode 100755 index 0000000..c51e2ff --- /dev/null +++ b/.coveragerc @@ -0,0 +1,39 @@ +# .coveragerc to control coverage.py +[run] +branch = True +omit = + ./docs/* + ./docker/* + ./examples/* + ./grammar/* + ./logs/* + ./tests/* + ./dist/* + ./images/* + ./output/* + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + # Don't complain about abstract methods, they aren't run: + @(abc\.)?abstractmethod + +ignore_errors = True + +[html] +directory = coverage_html_report \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index a364d12..d530d6d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ tox.ini test-requirements.txt README.md .stestr.conf -.pre-commit-config.yaml .gitignore logs/access.log .git/ diff --git a/poetry.lock b/poetry.lock index f13ffb7..e8d0430 100755 --- a/poetry.lock +++ b/poetry.lock @@ -111,6 +111,79 @@ files = [ {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, ] +[[package]] +name = "coverage" +version = "7.2.7" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +] + +[package.extras] +toml = ["tomli"] + [[package]] name = "distlib" version = "0.3.7" @@ -806,4 +879,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "f9a0b64ad4adc32e4fac061e17834c18b798eac5868c2a7aad1eeddd8d8c4e63" +content-hash = "604633e797026198f8c6db0da6791c4bfefe2a8e8501de135aa6c03bc76f669c" diff --git a/pyproject.toml b/pyproject.toml index 325aaf0..e75c34e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ types-docopt = "0.6.11.3" types-python-dateutil = "2.8.19.14" types-pytz = "2023.3.0.0" types-requests = "2.31.0.2" +coverage = "7.2.7" [build-system] requires = ["poetry-core>=1.0.0"] @@ -63,7 +64,6 @@ paths.source = [ "common", "ngsild", "sdmx2jsonld", - "tests", ] report.fail_under = 88 run.parallel = true @@ -80,16 +80,6 @@ module = [ "common", "ngsild", "sdmx2jsonld", - "tests", ] ignore_missing_imports = true warn_unused_ignores = true - -[tool.towncrier] -name = "tox" -filename = "docs/changelog.rst" -directory = "docs/changelog" -title_format = false -issue_format = ":issue:`{issue}`" -template = "docs/changelog/template.jinja2" -# possible types, all default: feature, bugfix, doc, removal, misc diff --git a/requirements-dev.txt b/test-requirements.txt similarity index 93% rename from requirements-dev.txt rename to test-requirements.txt index 65ad636..32f1147 100644 --- a/requirements-dev.txt +++ b/test-requirements.txt @@ -6,3 +6,4 @@ types-docopt==0.6.11.3 types-python-dateutil==2.8.19.14 types-pytz==2023.3.0.0 types-requests==2.31.0.2 +coverage==7.2.7 diff --git a/tox.ini b/tox.ini index cb476b6..ba8e7fa 100755 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,19 @@ [tox] requires = tox>=4 -env_list = lint, type, py{310, 311} +env_list = lint, type, py{310, 311}, coverage + +[testenv:coverage] +description = run coverage tests +deps = + pytest>=7 + pytest-sugar + nose + coverage +commands = + coverage erase + coverage run --source=./api,./cli,./common,./ngsild,./sdmx2jsonld --omit=./coverage_html_report,./dist,./docker,./docs,./examples,./images,./logs,./tests -m pytest + coverage report -m [testenv:py310] description = run unit tests in python3.10 From 816fea8a961357080df9666f2100505f6d244afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 25 Jul 2023 12:11:37 +0200 Subject: [PATCH 74/74] Update gitignore to add the environment configuration file for docker --- .gitignore | 1 - docker/.env | 13 +++++++++++++ tests/.env | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 docker/.env create mode 100755 tests/.env diff --git a/.gitignore b/.gitignore index 53796fa..1e3ef38 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,6 @@ celerybeat.pid *.sage.py # Environments -.env .venv env/ venv/ diff --git a/docker/.env b/docker/.env new file mode 100755 index 0000000..09be6ce --- /dev/null +++ b/docker/.env @@ -0,0 +1,13 @@ +# Project name +COMPOSE_PROJECT_NAME=fiware + +# Orion variables +ORION_LD_PORT=1026 +ORION_LD_VERSION=1.0.0 + +# MongoDB variables +MONGO_DB_PORT=27017 +MONGO_DB_VERSION=4.4 + +# IoTAgent-Turtle varaibles +IOTAGENT_PORT=5000 diff --git a/tests/.env b/tests/.env new file mode 100755 index 0000000..a9fa8f7 --- /dev/null +++ b/tests/.env @@ -0,0 +1,10 @@ +# Project name +COMPOSE_PROJECT_NAME=fiware + +# Orion variables +ORION_LD_PORT=1026 +ORION_LD_VERSION=1.0.0 + +# MongoDB variables +MONGO_DB_PORT=27017 +MONGO_DB_VERSION=4.4