[med-svn] [python-schema-salad] 04/07: New upstream version 2.6.20170630075932
Michael Crusoe
misterc-guest at moszumanska.debian.org
Tue Aug 8 14:46:16 UTC 2017
This is an automated email from the git hooks/post-receive script.
misterc-guest pushed a commit to branch master
in repository python-schema-salad.
commit 2d7cb0e0d5882a1ae86844dc3175144f281b4147
Author: Michael R. Crusoe <michael.crusoe at gmail.com>
Date: Wed Jul 5 01:49:22 2017 -0700
New upstream version 2.6.20170630075932
---
MANIFEST.in | 1 +
Makefile | 50 +--
PKG-INFO | 13 +-
README.rst | 7 +
schema_salad.egg-info/PKG-INFO | 13 +-
schema_salad.egg-info/SOURCES.txt | 25 +-
schema_salad.egg-info/entry_points.txt | 1 +
schema_salad.egg-info/pbr.json | 1 -
schema_salad.egg-info/requires.txt | 12 +-
schema_salad/__init__.py | 1 +
schema_salad/__main__.py | 1 +
schema_salad/add_dictlist.py | 8 -
schema_salad/aslist.py | 11 -
schema_salad/jsonld_context.py | 81 ++---
schema_salad/main.py | 48 +--
schema_salad/makedoc.py | 101 +++---
schema_salad/metaschema/map_res.yml | 36 +++
schema_salad/metaschema/map_res_proc.yml | 12 +
schema_salad/metaschema/map_res_schema.yml | 30 ++
schema_salad/metaschema/map_res_src.yml | 8 +
schema_salad/metaschema/metaschema.yml | 13 +
schema_salad/metaschema/metaschema2.yml | 317 ------------------
schema_salad/metaschema/metaschema2.yml~ | 318 -------------------
schema_salad/metaschema/metaschema_base.yml | 7 +
schema_salad/metaschema/salad.md | 19 +-
schema_salad/metaschema/typedsl_res.yml | 33 ++
schema_salad/metaschema/typedsl_res_proc.yml | 26 ++
schema_salad/metaschema/typedsl_res_schema.yml | 17 +
schema_salad/metaschema/typedsl_res_src.yml | 9 +
schema_salad/ref_resolver.py | 424 ++++++++++++++-----------
schema_salad/schema.py | 131 +++++---
schema_salad/sourceline.py | 49 +--
schema_salad/tests/.coverage | 1 -
schema_salad/tests/test_cli_args.py | 41 +++
schema_salad/tests/test_errors.py | 14 +-
schema_salad/tests/test_errors.py~ | 1 -
schema_salad/tests/test_examples.py | 25 +-
schema_salad/tests/test_fetch.py | 17 +-
schema_salad/tests/test_fetch.py~ | 13 -
schema_salad/tests/test_ref_resolver.py | 55 ++++
schema_salad/tests/test_schema/test12.cwl | 16 +
schema_salad/tests/test_schema/test13.cwl | 20 ++
schema_salad/tests/test_schema/test14.cwl | 11 +
schema_salad/tests/test_validate.pyx | 71 -----
schema_salad/tests/test_validate.py~ | 70 ----
schema_salad/tests/util.py | 4 +-
schema_salad/{flatten.py => utils.py} | 21 +-
schema_salad/validate.py | 132 +++++---
setup.cfg | 3 +-
setup.py | 30 +-
50 files changed, 1056 insertions(+), 1312 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index abcfe2a..c3870ab 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,4 +4,5 @@ include schema_salad/tests/test_schema/*.md
include schema_salad/tests/test_schema/*.yml
include schema_salad/tests/test_schema/*.cwl
include schema_salad/metaschema/*
+global-exclude *~
global-exclude *.pyc
diff --git a/Makefile b/Makefile
index f5a8240..c555c0a 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,8 @@ MODULE=schema_salad
# `[[` conditional expressions.
PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 pytest flake8
+COVBASE=coverage run --branch --append --source=${MODULE} \
+ --omit=schema_salad/tests/*
VERSION=$(shell git describe --tags --dirty | sed s/v//)
@@ -103,37 +105,31 @@ diff_pylint_report: pylint_report.txt
diff-quality --violations=pylint pylint_report.txt
.coverage: $(PYSOURCES)
- coverage run --branch --source=${MODULE} setup.py test
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ rm -f .coverage
+ $(COVBASE) setup.py test
+ $(COVBASE) -m schema_salad.main \
--print-jsonld-context schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-rdfs schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-avro schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-rdf schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-pre schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-index schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.main \
+ $(COVBASE) -m schema_salad.main \
--print-metadata schema_salad/metaschema/metaschema.yml \
> /dev/null
- coverage run --append --branch --source=${MODULE} \
- -m schema_salad.makedoc schema_salad/metaschema/metaschema.yml \
+ $(COVBASE) -m schema_salad.makedoc \
+ schema_salad/metaschema/metaschema.yml \
> /dev/null
coverage.xml: .coverage
@@ -169,12 +165,20 @@ list-author-emails:
@echo 'name, E-Mail Address'
@git log --format='%aN,%aE' | sort -u | grep -v 'root'
-mypy: ${PYSOURCES}
- rm -Rf typeshed/2.7/ruamel/yaml
+mypy2: ${PYSOURCES}
+ rm -Rf typeshed/2and3/ruamel/yaml
ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
- typeshed/2.7/ruamel/
- MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
- --fast-parser --warn-redundant-casts --warn-unused-ignores \
+ typeshed/2and3/ruamel/
+ MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+ --warn-redundant-casts --warn-unused-ignores \
+ schema_salad
+
+mypy3: ${PYSOURCES}
+ rm -Rf typeshed/2and3/ruamel/yaml
+ ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
+ typeshed/2and3/ruamel/
+ MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+ --warn-redundant-casts \
schema_salad
jenkins:
@@ -187,6 +191,6 @@ jenkins:
. env3/bin/activate ; \
pip install -U setuptools pip wheel ; \
${MAKE} install-dep ; \
- pip install -U mypy-lang typed-ast ; ${MAKE} mypy
+ pip install -U -r mypy_requirements.txt ; ${MAKE} mypy
FORCE:
diff --git a/PKG-INFO b/PKG-INFO
index c53e8d8..b4230c3 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: schema-salad
-Version: 2.2.20170119151016
+Version: 2.6.20170630075932
Summary: Schema Annotations for Linked Avro Data (SALAD)
Home-page: https://github.com/common-workflow-language/common-workflow-language
Author: Common workflow language working group
@@ -36,6 +36,13 @@ Description: Schema Salad
$ python
>>> import schema_salad
+ To install from source::
+
+ git clone https://github.com/common-workflow-language/schema_salad
+ cd schema_salad
+ python setup.py install
+
+
Documentation
-------------
@@ -90,3 +97,7 @@ Classifier: Operating System :: POSIX :: Linux
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Development Status :: 4 - Beta
Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
diff --git a/README.rst b/README.rst
index e66427c..38ffb42 100644
--- a/README.rst
+++ b/README.rst
@@ -27,6 +27,13 @@ Usage
$ python
>>> import schema_salad
+To install from source::
+
+ git clone https://github.com/common-workflow-language/schema_salad
+ cd schema_salad
+ python setup.py install
+
+
Documentation
-------------
diff --git a/schema_salad.egg-info/PKG-INFO b/schema_salad.egg-info/PKG-INFO
index c53e8d8..b4230c3 100644
--- a/schema_salad.egg-info/PKG-INFO
+++ b/schema_salad.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: schema-salad
-Version: 2.2.20170119151016
+Version: 2.6.20170630075932
Summary: Schema Annotations for Linked Avro Data (SALAD)
Home-page: https://github.com/common-workflow-language/common-workflow-language
Author: Common workflow language working group
@@ -36,6 +36,13 @@ Description: Schema Salad
$ python
>>> import schema_salad
+ To install from source::
+
+ git clone https://github.com/common-workflow-language/schema_salad
+ cd schema_salad
+ python setup.py install
+
+
Documentation
-------------
@@ -90,3 +97,7 @@ Classifier: Operating System :: POSIX :: Linux
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Development Status :: 4 - Beta
Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
diff --git a/schema_salad.egg-info/SOURCES.txt b/schema_salad.egg-info/SOURCES.txt
index 73a6c34..7c15ca2 100644
--- a/schema_salad.egg-info/SOURCES.txt
+++ b/schema_salad.egg-info/SOURCES.txt
@@ -6,21 +6,18 @@ setup.cfg
setup.py
schema_salad/__init__.py
schema_salad/__main__.py
-schema_salad/add_dictlist.py
-schema_salad/aslist.py
-schema_salad/flatten.py
schema_salad/jsonld_context.py
schema_salad/main.py
schema_salad/makedoc.py
schema_salad/ref_resolver.py
schema_salad/schema.py
schema_salad/sourceline.py
+schema_salad/utils.py
schema_salad/validate.py
schema_salad.egg-info/PKG-INFO
schema_salad.egg-info/SOURCES.txt
schema_salad.egg-info/dependency_links.txt
schema_salad.egg-info/entry_points.txt
-schema_salad.egg-info/pbr.json
schema_salad.egg-info/requires.txt
schema_salad.egg-info/top_level.txt
schema_salad.egg-info/zip-safe
@@ -37,28 +34,31 @@ schema_salad/metaschema/link_res.yml
schema_salad/metaschema/link_res_proc.yml
schema_salad/metaschema/link_res_schema.yml
schema_salad/metaschema/link_res_src.yml
+schema_salad/metaschema/map_res.yml
+schema_salad/metaschema/map_res_proc.yml
+schema_salad/metaschema/map_res_schema.yml
+schema_salad/metaschema/map_res_src.yml
schema_salad/metaschema/metaschema.yml
-schema_salad/metaschema/metaschema2.yml
-schema_salad/metaschema/metaschema2.yml~
schema_salad/metaschema/metaschema_base.yml
schema_salad/metaschema/salad.md
+schema_salad/metaschema/typedsl_res.yml
+schema_salad/metaschema/typedsl_res_proc.yml
+schema_salad/metaschema/typedsl_res_schema.yml
+schema_salad/metaschema/typedsl_res_src.yml
schema_salad/metaschema/vocab_res.yml
schema_salad/metaschema/vocab_res_proc.yml
schema_salad/metaschema/vocab_res_schema.yml
schema_salad/metaschema/vocab_res_src.yml
-schema_salad/tests/.coverage
schema_salad/tests/EDAM.owl
schema_salad/tests/Process.yml
schema_salad/tests/__init__.py
schema_salad/tests/frag.yml
schema_salad/tests/mixin.yml
+schema_salad/tests/test_cli_args.py
schema_salad/tests/test_errors.py
-schema_salad/tests/test_errors.py~
schema_salad/tests/test_examples.py
schema_salad/tests/test_fetch.py
-schema_salad/tests/test_fetch.py~
-schema_salad/tests/test_validate.pyx
-schema_salad/tests/test_validate.py~
+schema_salad/tests/test_ref_resolver.py
schema_salad/tests/util.py
schema_salad/tests/test_schema/CommandLineTool.yml
schema_salad/tests/test_schema/CommonWorkflowLanguage.yml
@@ -72,6 +72,9 @@ schema_salad/tests/test_schema/metaschema_base.yml
schema_salad/tests/test_schema/test1.cwl
schema_salad/tests/test_schema/test10.cwl
schema_salad/tests/test_schema/test11.cwl
+schema_salad/tests/test_schema/test12.cwl
+schema_salad/tests/test_schema/test13.cwl
+schema_salad/tests/test_schema/test14.cwl
schema_salad/tests/test_schema/test2.cwl
schema_salad/tests/test_schema/test3.cwl
schema_salad/tests/test_schema/test4.cwl
diff --git a/schema_salad.egg-info/entry_points.txt b/schema_salad.egg-info/entry_points.txt
index 9cd6059..c055407 100644
--- a/schema_salad.egg-info/entry_points.txt
+++ b/schema_salad.egg-info/entry_points.txt
@@ -1,3 +1,4 @@
[console_scripts]
+schema-salad-doc = schema_salad.makedoc:main
schema-salad-tool = schema_salad.main:main
diff --git a/schema_salad.egg-info/pbr.json b/schema_salad.egg-info/pbr.json
deleted file mode 100644
index ded4ec4..0000000
--- a/schema_salad.egg-info/pbr.json
+++ /dev/null
@@ -1 +0,0 @@
-{"is_release": false, "git_version": "d21fa84"}
\ No newline at end of file
diff --git a/schema_salad.egg-info/requires.txt b/schema_salad.egg-info/requires.txt
index a27d4da..2e4673f 100644
--- a/schema_salad.egg-info/requires.txt
+++ b/schema_salad.egg-info/requires.txt
@@ -1,10 +1,16 @@
setuptools
requests >= 1.0
-ruamel.yaml >= 0.12.4
-rdflib >= 4.2.0, < 4.3.0
+ruamel.yaml >= 0.12.4, < 0.15
+rdflib >= 4.2.2, < 4.3.0
rdflib-jsonld >= 0.3.0, < 0.5.0
mistune >= 0.7.3, < 0.8
-typing >= 3.5.2, < 3.6
+typing >= 3.5.3
CacheControl >= 0.11.7, < 0.12
lockfile >= 0.9
+six >= 1.8.0
+
+[:python_version<"3"]
avro
+
+[:python_version>="3"]
+avro-python3
diff --git a/schema_salad/__init__.py b/schema_salad/__init__.py
index 381ec76..a751d64 100644
--- a/schema_salad/__init__.py
+++ b/schema_salad/__init__.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import logging
import sys
import typing
diff --git a/schema_salad/__main__.py b/schema_salad/__main__.py
index 4bf3d7e..5890f6f 100644
--- a/schema_salad/__main__.py
+++ b/schema_salad/__main__.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
from . import main
import sys
import typing
diff --git a/schema_salad/add_dictlist.py b/schema_salad/add_dictlist.py
deleted file mode 100644
index 711f580..0000000
--- a/schema_salad/add_dictlist.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import sys
-from typing import Any, Dict
-
-
-def add_dictlist(di, key, val): # type: (Dict, Any, Any) -> None
- if key not in di:
- di[key] = []
- di[key].append(val)
diff --git a/schema_salad/aslist.py b/schema_salad/aslist.py
deleted file mode 100644
index 27602ab..0000000
--- a/schema_salad/aslist.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import sys
-from typing import Any, List
-
-
-def aslist(l): # type: (Any) -> List
- """Convenience function to wrap single items and lists, and return lists unchanged."""
-
- if isinstance(l, list):
- return l
- else:
- return [l]
diff --git a/schema_salad/jsonld_context.py b/schema_salad/jsonld_context.py
index 7141b07..57e6ff4 100755
--- a/schema_salad/jsonld_context.py
+++ b/schema_salad/jsonld_context.py
@@ -1,6 +1,11 @@
+from __future__ import absolute_import
import collections
import shutil
import json
+
+import six
+from six.moves import urllib
+
import ruamel.yaml as yaml
try:
from ruamel.yaml import CSafeLoader as SafeLoader
@@ -16,38 +21,38 @@ import rdflib
from rdflib import Graph, URIRef
import rdflib.namespace
from rdflib.namespace import RDF, RDFS
-import urlparse
import logging
-from .aslist import aslist
-from typing import Any, cast, Dict, Iterable, Tuple, Union
+from schema_salad.utils import aslist
+from typing import (cast, Any, Dict, Iterable, List, Optional, Text, Tuple,
+ Union)
from .ref_resolver import Loader, ContextType
_logger = logging.getLogger("salad")
def pred(datatype, # type: Dict[str, Union[Dict, str]]
- field, # type: Dict
+ field, # type: Optional[Dict]
name, # type: str
context, # type: ContextType
defaultBase, # type: str
namespaces # type: Dict[str, rdflib.namespace.Namespace]
):
- # type: (...) -> Union[Dict, str]
- split = urlparse.urlsplit(name)
+ # type: (...) -> Union[Dict, Text]
+ split = urllib.parse.urlsplit(name)
- vee = None # type: Union[str, unicode]
+ vee = None # type: Optional[Text]
- if split.scheme:
+ if split.scheme != '':
vee = name
- (ns, ln) = rdflib.namespace.split_uri(unicode(vee))
+ (ns, ln) = rdflib.namespace.split_uri(six.text_type(vee))
name = ln
if ns[0:-1] in namespaces:
- vee = unicode(namespaces[ns[0:-1]][ln])
+ vee = six.text_type(namespaces[ns[0:-1]][ln])
_logger.debug("name, v %s %s", name, vee)
- v = None # type: Any
+ v = None # type: Optional[Dict]
- if field and "jsonldPredicate" in field:
+ if field is not None and "jsonldPredicate" in field:
if isinstance(field["jsonldPredicate"], dict):
v = {}
for k, val in field["jsonldPredicate"].items():
@@ -68,11 +73,6 @@ def pred(datatype, # type: Dict[str, Union[Dict, str]]
"Dictionaries")
else:
raise Exception("jsonldPredicate must be a List of Dictionaries.")
- # if not v:
- # if field and "jsonldPrefix" in field:
- # defaultBase = field["jsonldPrefix"]
- # elif "jsonldPrefix" in datatype:
- # defaultBase = datatype["jsonldPrefix"]
ret = v or vee
@@ -106,15 +106,15 @@ def process_type(t, # type: Dict[str, Any]
classnode = URIRef(recordname)
g.add((classnode, RDF.type, RDFS.Class))
- split = urlparse.urlsplit(recordname)
- if "jsonldPrefix" in t:
- predicate = "%s:%s" % (t["jsonldPrefix"], recordname)
- elif split.scheme:
- (ns, ln) = rdflib.namespace.split_uri(unicode(recordname))
- predicate = recordname
- recordname = ln
- else:
- predicate = "%s:%s" % (defaultPrefix, recordname)
+ split = urllib.parse.urlsplit(recordname)
+ predicate = recordname
+ if t.get("inVocab", True):
+ if split.scheme:
+ (ns, ln) = rdflib.namespace.split_uri(six.text_type(recordname))
+ predicate = recordname
+ recordname = ln
+ else:
+ predicate = "%s:%s" % (defaultPrefix, recordname)
if context.get(recordname, predicate) != predicate:
raise Exception("Predicate collision on '%s', '%s' != '%s'" % (
@@ -132,15 +132,16 @@ def process_type(t, # type: Dict[str, Any]
_logger.debug("Processing field %s", i)
- v = pred(t, i, fieldname, context, defaultPrefix, namespaces)
+ v = pred(t, i, fieldname, context, defaultPrefix,
+ namespaces) # type: Union[Dict[Any, Any], Text, None]
- if isinstance(v, basestring):
+ if isinstance(v, six.string_types):
v = v if v[0] != "@" else None
- else:
+ elif v is not None:
v = v["_ at id"] if v.get("_ at id", "@")[0] != "@" else None
- if v:
- (ns, ln) = rdflib.namespace.split_uri(unicode(v))
+ if bool(v):
+ (ns, ln) = rdflib.namespace.split_uri(six.text_type(v))
if ns[0:-1] in namespaces:
propnode = namespaces[ns[0:-1]][ln]
else:
@@ -191,8 +192,8 @@ def salad_to_jsonld_context(j, schema_ctx):
return (context, g)
-def fix_jsonld_ids(obj, # type: Union[Dict[unicode, Any], List[Dict[unicode, Any]]]
- ids # type: List[unicode]
+def fix_jsonld_ids(obj, # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
+ ids # type: List[Text]
):
# type: (...) -> None
if isinstance(obj, dict):
@@ -206,22 +207,22 @@ def fix_jsonld_ids(obj, # type: Union[Dict[unicode, Any], List[Dict[unicode,
fix_jsonld_ids(entry, ids)
-def makerdf(workflow, # type: Union[str, unicode]
- wf, # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
+def makerdf(workflow, # type: Text
+ wf, # type: Union[List[Dict[Text, Any]], Dict[Text, Any]]
ctx, # type: ContextType
graph=None # type: Graph
):
# type: (...) -> Graph
prefixes = {}
idfields = []
- for k, v in ctx.iteritems():
+ for k, v in six.iteritems(ctx):
if isinstance(v, dict):
url = v["@id"]
else:
url = v
if url == "@id":
idfields.append(k)
- doc_url, frg = urlparse.urldefrag(url)
+ doc_url, frg = urllib.parse.urldefrag(url)
if "/" in frg:
p = frg.split("/")[0]
prefixes[p] = u"%s#%s/" % (doc_url, p)
@@ -236,16 +237,16 @@ def makerdf(workflow, # type: Union[str, unicode]
if isinstance(wf, list):
for w in wf:
w["@context"] = ctx
- g.parse(data=json.dumps(w), format='json-ld', location=workflow)
+ g.parse(data=json.dumps(w), format='json-ld', publicID=str(workflow))
else:
wf["@context"] = ctx
- g.parse(data=json.dumps(wf), format='json-ld', location=workflow)
+ g.parse(data=json.dumps(wf), format='json-ld', publicID=str(workflow))
# Bug in json-ld loader causes @id fields to be added to the graph
for sub, pred, obj in g.triples((None, URIRef("@id"), None)):
g.remove((sub, pred, obj))
- for k2, v2 in prefixes.iteritems():
+ for k2, v2 in six.iteritems(prefixes):
g.namespace_manager.bind(k2, v2)
return g
diff --git a/schema_salad/main.py b/schema_salad/main.py
index f51184b..33225aa 100644
--- a/schema_salad/main.py
+++ b/schema_salad/main.py
@@ -1,15 +1,17 @@
from __future__ import print_function
+from __future__ import absolute_import
import argparse
import logging
import sys
import traceback
import json
import os
-import urlparse
+
+from six.moves import urllib
import pkg_resources # part of setuptools
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Union, Text
from rdflib import Graph, plugin
from rdflib.serializer import Serializer
@@ -20,6 +22,7 @@ from . import makedoc
from . import validate
from .sourceline import strip_dup_lineno
from .ref_resolver import Loader
+import six
_logger = logging.getLogger("salad")
@@ -28,8 +31,8 @@ register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser')
def printrdf(workflow, # type: str
- wf, # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
- ctx, # type: Dict[unicode, Any]
+ wf, # type: Union[List[Dict[Text, Any]], Dict[Text, Any]]
+ ctx, # type: Dict[Text, Any]
sr # type: str
):
# type: (...) -> None
@@ -62,8 +65,6 @@ def main(argsl=None): # type: (List[str]) -> int
"--print-index", action="store_true", help="Print node index")
exgroup.add_argument("--print-metadata",
action="store_true", help="Print document metadata")
- exgroup.add_argument("--version", action="store_true",
- help="Print version")
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)",
@@ -79,11 +80,18 @@ def main(argsl=None): # type: (List[str]) -> int
exgroup.add_argument("--debug", action="store_true",
help="Print even more logging")
- parser.add_argument("schema", type=str)
+ parser.add_argument("schema", type=str, nargs="?", default=None)
parser.add_argument("document", type=str, nargs="?", default=None)
+ parser.add_argument("--version", "-v", action="store_true",
+ help="Print version", default=None)
+
args = parser.parse_args(argsl)
+ if args.version is None and args.schema is None:
+ print('%s: error: too few arguments' % sys.argv[0])
+ return 1
+
if args.quiet:
_logger.setLevel(logging.WARN)
if args.debug:
@@ -92,10 +100,10 @@ def main(argsl=None): # type: (List[str]) -> int
pkg = pkg_resources.require("schema_salad")
if pkg:
if args.version:
- print("%s %s" % (sys.argv[0], pkg[0].version))
+ print("%s Current version: %s" % (sys.argv[0], pkg[0].version))
return 0
else:
- _logger.info("%s %s", sys.argv[0], pkg[0].version)
+ _logger.info("%s Current version: %s", sys.argv[0], pkg[0].version)
# Get the metaschema to validate the schema
metaschema_names, metaschema_doc, metaschema_loader = schema.get_metaschema()
@@ -103,7 +111,7 @@ def main(argsl=None): # type: (List[str]) -> int
# Load schema document and resolve refs
schema_uri = args.schema
- if not urlparse.urlparse(schema_uri)[0]:
+ if not urllib.parse.urlparse(schema_uri)[0]:
schema_uri = "file://" + os.path.abspath(schema_uri)
schema_raw_doc = metaschema_loader.fetch(schema_uri)
@@ -113,8 +121,8 @@ def main(argsl=None): # type: (List[str]) -> int
except (validate.ValidationException) as e:
_logger.error("Schema `%s` failed link checking:\n%s",
args.schema, e, exc_info=(True if args.debug else False))
- _logger.debug("Index is %s", metaschema_loader.idx.keys())
- _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
+ _logger.debug("Index is %s", list(metaschema_loader.idx.keys()))
+ _logger.debug("Vocabulary is %s", list(metaschema_loader.vocab.keys()))
return 1
except (RuntimeError) as e:
_logger.error("Schema `%s` read error:\n%s",
@@ -127,14 +135,14 @@ def main(argsl=None): # type: (List[str]) -> int
return 0
if not args.document and args.print_index:
- print(json.dumps(metaschema_loader.idx.keys(), indent=4))
+ print(json.dumps(list(metaschema_loader.idx.keys()), indent=4))
return 0
# Validate the schema document against the metaschema
try:
schema.validate_doc(metaschema_names, schema_doc,
metaschema_loader, args.strict,
- source=schema_metadata["name"])
+ source=schema_metadata.get("name"))
except validate.ValidationException as e:
_logger.error("While validating schema `%s`:\n%s" %
(args.schema, str(e)))
@@ -146,7 +154,11 @@ def main(argsl=None): # type: (List[str]) -> int
metactx = schema_raw_doc.get("$namespaces", {})
if "$base" in schema_raw_doc:
metactx["@base"] = schema_raw_doc["$base"]
- (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(schema_doc, metactx)
+ if schema_doc is not None:
+ (schema_ctx, rdfs) = jsonld_context.salad_to_jsonld_context(
+ schema_doc, metactx)
+ else:
+ raise Exception("schema_doc is None??")
# Create the loader that will be used to load the target document.
document_loader = Loader(schema_ctx)
@@ -196,12 +208,12 @@ def main(argsl=None): # type: (List[str]) -> int
# Load target document and resolve refs
try:
uri = args.document
- if not urlparse.urlparse(uri)[0]:
+ if not urllib.parse.urlparse(uri)[0]:
doc = "file://" + os.path.abspath(uri)
document, doc_metadata = document_loader.resolve_ref(uri)
except (validate.ValidationException, RuntimeError) as e:
_logger.error("Document `%s` failed validation:\n%s",
- args.document, strip_dup_lineno(unicode(e)), exc_info=args.debug)
+ args.document, strip_dup_lineno(six.text_type(e)), exc_info=args.debug)
return 1
# Optionally print the document after ref resolution
@@ -210,7 +222,7 @@ def main(argsl=None): # type: (List[str]) -> int
return 0
if args.print_index:
- print(json.dumps(document_loader.idx.keys(), indent=4))
+ print(json.dumps(list(document_loader.idx.keys()), indent=4))
return 0
# Validate the schema document against the metaschema
diff --git a/schema_salad/makedoc.py b/schema_salad/makedoc.py
index 0325ad8..c0e7c8a 100644
--- a/schema_salad/makedoc.py
+++ b/schema_salad/makedoc.py
@@ -1,23 +1,27 @@
+from __future__ import absolute_import
+
import mistune
-from . import schema
+import argparse
import json
import os
import copy
import re
import sys
-from StringIO import StringIO
import logging
-import urlparse
-from .aslist import aslist
-from .add_dictlist import add_dictlist
-import re
-import argparse
-from typing import Any, IO, Union
+
+from . import schema
+from .utils import add_dictlist, aslist
+
+import six
+from six.moves import range
+from six.moves import urllib
+from six import StringIO
+from typing import cast, Any, Dict, IO, List, Optional, Set, Text, Union
_logger = logging.getLogger("salad")
-def has_types(items): # type: (Any) -> List[basestring]
+def has_types(items): # type: (Any) -> List[Text]
r = [] # type: List
if isinstance(items, dict):
if items["type"] == "https://w3id.org/cwl/salad#record":
@@ -30,13 +34,13 @@ def has_types(items): # type: (Any) -> List[basestring]
for i in items:
r.extend(has_types(i))
return r
- if isinstance(items, basestring):
+ if isinstance(items, six.string_types):
return [items]
return []
-def linkto(item):
- _, frg = urlparse.urldefrag(item)
+def linkto(item): # type: (Text) -> Text
+ _, frg = urllib.parse.urldefrag(item)
return "[%s](#%s)" % (frg, to_id(frg))
@@ -46,17 +50,17 @@ class MyRenderer(mistune.Renderer):
super(mistune.Renderer, self).__init__()
self.options = {}
- def header(self, text, level, raw=None):
+ def header(self, text, level, raw=None): # type: (Text, int, Any) -> Text
return """<h%i id="%s">%s</h%i>""" % (level, to_id(text), text, level)
- def table(self, header, body):
+ def table(self, header, body): # type: (Text, Text) -> Text
return (
'<table class="table table-striped">\n<thead>%s</thead>\n'
'<tbody>\n%s</tbody>\n</table>\n'
) % (header, body)
-def to_id(text): # type: (Union[str, unicode]) -> Union[str, unicode]
+def to_id(text): # type: (Text) -> Text
textid = text
if text[0] in ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"):
try:
@@ -136,7 +140,7 @@ def number_headings(toc, maindoc): # type: (ToC, str) -> str
if not skip:
m = re.match(r'^(#+) (.*)', line)
- if m:
+ if m is not None:
num = toc.add_entry(len(m.group(1)), m.group(2))
line = "%s %s %s" % (m.group(1), num, m.group(2))
line = re.sub(r'^(https?://\S+)', r'[\1](\1)', line)
@@ -158,8 +162,8 @@ def fix_doc(doc): # type: (Union[List[str], str]) -> str
class RenderType(object):
- def __init__(self, toc, j, renderlist, redirects):
- # type: (ToC, List[Dict], str, Dict) -> None
+ def __init__(self, toc, j, renderlist, redirects, primitiveType):
+ # type: (ToC, List[Dict], str, Dict, str) -> None
self.typedoc = StringIO()
self.toc = toc
self.subs = {} # type: Dict[str, str]
@@ -167,7 +171,8 @@ class RenderType(object):
self.docAfter = {} # type: Dict[str, List]
self.rendered = set() # type: Set[str]
self.redirects = redirects
- self.title = None # type: str
+ self.title = None # type: Optional[str]
+ self.primitiveType = primitiveType
for t in j:
if "extends" in t:
@@ -203,8 +208,8 @@ class RenderType(object):
if tp not in self.uses:
self.uses[tp] = []
if (t["name"], f["name"]) not in self.uses[tp]:
- _, frg1 = urlparse.urldefrag(t["name"])
- _, frg2 = urlparse.urldefrag(f["name"])
+ _, frg1 = urllib.parse.urldefrag(t["name"])
+ _, frg2 = urllib.parse.urldefrag(f["name"])
self.uses[tp].append((frg1, frg2))
if tp not in basicTypes and tp not in self.record_refs[t["name"]]:
self.record_refs[t["name"]].append(tp)
@@ -224,10 +229,9 @@ class RenderType(object):
tp, # type: Any
redirects, # type: Dict[str, str]
nbsp=False, # type: bool
- jsonldPredicate=None # type: Dict[str, str]
+ jsonldPredicate=None # type: Optional[Dict[str, str]]
):
- # type: (...) -> Union[str, unicode]
- global primitiveType
+ # type: (...) -> Text
if isinstance(tp, list):
if nbsp and len(tp) <= 3:
return " | ".join([self.typefmt(n, redirects, jsonldPredicate=jsonldPredicate) for n in tp])
@@ -237,7 +241,7 @@ class RenderType(object):
if tp["type"] == "https://w3id.org/cwl/salad#array":
ar = "array<%s>" % (self.typefmt(
tp["items"], redirects, nbsp=True))
- if jsonldPredicate and "mapSubject" in jsonldPredicate:
+ if jsonldPredicate is not None and "mapSubject" in jsonldPredicate:
if "mapPredicate" in jsonldPredicate:
ar += " | map<%s.%s, %s.%s>" % (self.typefmt(tp["items"], redirects),
jsonldPredicate[
@@ -251,7 +255,7 @@ class RenderType(object):
self.typefmt(tp["items"], redirects))
return ar
if tp["type"] in ("https://w3id.org/cwl/salad#record", "https://w3id.org/cwl/salad#enum"):
- frg = schema.avro_name(tp["name"])
+ frg = cast(Text, schema.avro_name(tp["name"]))
if tp["name"] in redirects:
return """<a href="%s">%s</a>""" % (redirects[tp["name"]], frg)
elif tp["name"] in self.typemap:
@@ -264,18 +268,22 @@ class RenderType(object):
if str(tp) in redirects:
return """<a href="%s">%s</a>""" % (redirects[tp], redirects[tp])
elif str(tp) in basicTypes:
- return """<a href="%s">%s</a>""" % (primitiveType, schema.avro_name(str(tp)))
+ return """<a href="%s">%s</a>""" % (self.primitiveType, schema.avro_name(str(tp)))
else:
- _, frg = urlparse.urldefrag(tp)
- if frg:
+ _, frg = urllib.parse.urldefrag(tp)
+ if frg is not '':
tp = frg
return """<a href="#%s">%s</a>""" % (to_id(tp), tp)
+ raise Exception("We should not be here!")
def render_type(self, f, depth): # type: (Dict[str, Any], int) -> None
if f["name"] in self.rendered or f["name"] in self.redirects:
return
self.rendered.add(f["name"])
+ if f.get("abstract"):
+ return
+
if "doc" not in f:
f["doc"] = ""
@@ -321,16 +329,18 @@ class RenderType(object):
lines.append(l)
f["doc"] = "\n".join(lines)
- _, frg = urlparse.urldefrag(f["name"])
+ _, frg = urllib.parse.urldefrag(f["name"])
num = self.toc.add_entry(depth, frg)
- doc = "## %s %s\n" % (num, frg)
+ doc = u"%s %s %s\n" % (("#" * depth), num, frg)
else:
- doc = ""
+ doc = u""
if self.title is None and f["doc"]:
- self.title = f["doc"][0:f["doc"].index("\n")]
- if self.title.startswith('# '):
- self.title = self.title[2:]
+ title = f["doc"][0:f["doc"].index("\n")]
+ if title.startswith('# '):
+ self.title = title[2:]
+ else:
+ self.title = title
if f["type"] == "documentation":
f["doc"] = number_headings(self.toc, f["doc"])
@@ -410,13 +420,13 @@ class RenderType(object):
self.render_type(self.typemap[s], depth)
-def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
- # type: (List[Dict[unicode, Any]], IO[Any], str, Dict, str, str) -> None
+def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink, primtype):
+ # type: (List[Dict[Text, Any]], IO[Any], str, Dict, str, str, str) -> None
toc = ToC()
toc.start_numbering = False
- rt = RenderType(toc, j, renderlist, redirects)
- content = rt.typedoc.getvalue() # type: unicode
+ rt = RenderType(toc, j, renderlist, redirects, primtype)
+ content = rt.typedoc.getvalue() # type: Text
outdoc.write("""
<!DOCTYPE html>
@@ -493,8 +503,7 @@ def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
</html>""")
-if __name__ == "__main__":
-
+def main(): # type: () -> None
parser = argparse.ArgumentParser()
parser.add_argument("schema")
parser.add_argument('--only', action='append')
@@ -505,7 +514,7 @@ if __name__ == "__main__":
args = parser.parse_args()
- s = [] # type: List[Dict[unicode, Any]]
+ s = [] # type: List[Dict[Text, Any]]
a = args.schema
with open(a) as f:
if a.endswith("md"):
@@ -523,10 +532,12 @@ if __name__ == "__main__":
s.append(j)
else:
raise ValueError("Schema must resolve to a list or a dict")
-
- primitiveType = args.primtype
redirect = {}
for r in (args.redirect or []):
redirect[r.split("=")[0]] = r.split("=")[1]
renderlist = args.only if args.only else []
- avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink)
+ avrold_doc(s, sys.stdout, renderlist, redirect, args.brand, args.brandlink, args.primtype)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/schema_salad/metaschema/map_res.yml b/schema_salad/metaschema/map_res.yml
new file mode 100644
index 0000000..bbcee48
--- /dev/null
+++ b/schema_salad/metaschema/map_res.yml
@@ -0,0 +1,36 @@
+- |
+ ## Identifier maps
+
+ The schema may designate certain fields as having a `mapSubject`. If the
+ value of the field is a JSON object, it must be transformed into an array of
+ JSON objects. Each key-value pair from the source JSON object is a list
+ item, each list item must be a JSON objects, and the value of the key is
+ assigned to the field specified by `mapSubject`.
+
+ Fields which have `mapSubject` specified may also supply a `mapPredicate`.
+ If the value of a map item is not a JSON object, the item is transformed to a
+ JSON object with the key assigned to the field specified by `mapSubject` and
+ the value assigned to the field specified by `mapPredicate`.
+
+ ### Identifier map example
+
+ Given the following schema:
+
+ ```
+- $include: map_res_schema.yml
+- |
+ ```
+
+ Process the following example:
+
+ ```
+- $include: map_res_src.yml
+- |
+ ```
+
+ This becomes:
+
+ ```
+- $include: map_res_proc.yml
+- |
+ ```
diff --git a/schema_salad/metaschema/map_res_proc.yml b/schema_salad/metaschema/map_res_proc.yml
new file mode 100644
index 0000000..52e9c22
--- /dev/null
+++ b/schema_salad/metaschema/map_res_proc.yml
@@ -0,0 +1,12 @@
+{
+ "mapped": [
+ {
+ "value": "daphne",
+ "key": "fred"
+ },
+ {
+ "value": "scooby",
+ "key": "shaggy"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/schema_salad/metaschema/map_res_schema.yml b/schema_salad/metaschema/map_res_schema.yml
new file mode 100644
index 0000000..086cc29
--- /dev/null
+++ b/schema_salad/metaschema/map_res_schema.yml
@@ -0,0 +1,30 @@
+{
+ "$graph": [{
+ "name": "MappedType",
+ "type": "record",
+ "documentRoot": true,
+ "fields": [{
+ "name": "mapped",
+ "type": {
+ "type": "array",
+ "items": "ExampleRecord"
+ },
+ "jsonldPredicate": {
+ "mapSubject": "key",
+ "mapPredicate": "value"
+ }
+ }],
+ },
+ {
+ "name": "ExampleRecord",
+ "type": "record",
+ "fields": [{
+ "name": "key",
+ "type": "string"
+ }, {
+ "name": "value",
+ "type": "string"
+ }
+ ]
+ }]
+}
diff --git a/schema_salad/metaschema/map_res_src.yml b/schema_salad/metaschema/map_res_src.yml
new file mode 100644
index 0000000..9df0c35
--- /dev/null
+++ b/schema_salad/metaschema/map_res_src.yml
@@ -0,0 +1,8 @@
+{
+ "mapped": {
+ "shaggy": {
+ "value": "scooby"
+ },
+ "fred": "daphne"
+ }
+}
\ No newline at end of file
diff --git a/schema_salad/metaschema/metaschema.yml b/schema_salad/metaschema/metaschema.yml
index d5472e9..28b9e66 100644
--- a/schema_salad/metaschema/metaschema.yml
+++ b/schema_salad/metaschema/metaschema.yml
@@ -18,6 +18,8 @@ $graph:
- $import: link_res.yml
- $import: vocab_res.yml
- $include: import_include.md
+ - $import: map_res.yml
+ - $import: typedsl_res.yml
- name: "Link_Validation"
type: documentation
@@ -154,16 +156,24 @@ $graph:
- name: NamedType
type: record
abstract: true
+ docParent: "#Schema"
fields:
- name: name
type: string
jsonldPredicate: "@id"
doc: "The identifier for this type"
+ - name: inVocab
+ type: boolean?
+ doc: |
+ By default or if "true", include the short name of this type in the
+ vocabulary (the keys of the JSON-LD context). If false, do not include
+ the short name in the vocabulary.
- name: DocType
type: record
abstract: true
+ docParent: "#Schema"
fields:
- name: doc
type:
@@ -240,6 +250,7 @@ $graph:
- name: SaladRecordSchema
+ docParent: "#Schema"
type: record
extends: [NamedType, RecordSchema, SchemaDefinedType]
documentRoot: true
@@ -277,6 +288,7 @@ $graph:
mapPredicate: specializeTo
- name: SaladEnumSchema
+ docParent: "#Schema"
type: record
extends: [EnumSchema, SchemaDefinedType]
documentRoot: true
@@ -297,6 +309,7 @@ $graph:
- name: Documentation
type: record
+ docParent: "#Schema"
extends: [NamedType, DocType]
documentRoot: true
doc: |
diff --git a/schema_salad/metaschema/metaschema2.yml b/schema_salad/metaschema/metaschema2.yml
deleted file mode 100644
index c928928..0000000
--- a/schema_salad/metaschema/metaschema2.yml
+++ /dev/null
@@ -1,317 +0,0 @@
-$base: "https://w3id.org/cwl/salad#"
-
-$namespaces:
- sld: "https://w3id.org/cwl/salad#"
- dct: "http://purl.org/dc/terms/"
- rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
- rdfs: "http://www.w3.org/2000/01/rdf-schema#"
- xsd: "http://www.w3.org/2001/XMLSchema#"
-
-$graph:
-
-- name: "Semantic_Annotations_for_Linked_Avro_Data"
- type: documentation
- doc:
- - $include: salad.md
- - $import: field_name.yml
- - $import: ident_res.yml
- - $import: link_res.yml
- - $import: vocab_res.yml
- - $include: import_include.md
-
-- name: "Link_Validation"
- type: documentation
- doc: |
- # Link validation
-
- Once a document has been preprocessed, an implementation may validate
- links. The link validation traversal may visit fields which the schema
- designates as link fields and check that each URI references an existing
- object in the current document, an imported document, file system, or
- network resource. Failure to validate links may be a fatal error. Link
- validation behavior for individual fields may be modified by `identity` and
- `noLinkCheck` in the `jsonldPredicate` section of the field schema.
-
-
-- name: "Schema_validation"
- type: documentation
- doc: ""
-
-
-# - name: "JSON_LD_Context"
-# type: documentation
-# doc: |
-# # Generating JSON-LD Context
-
-# How to generate the json-ld context...
-
-
-- $import: metaschema_base.yml
-
-- name: JsonldPredicate
- type: record
- doc: |
- Attached to a record field to define how the parent record field is handled for
- URI resolution and JSON-LD context generation.
- fields:
- - name: _id
- type: string?
- jsonldPredicate:
- _id: sld:_id
- _type: "@id"
- identity: true
- doc: |
- The predicate URI that this field corresponds to.
- Corresponds to JSON-LD `@id` directive.
- - name: _type
- type: string?
- doc: |
- The context type hint, corresponds to JSON-LD `@type` directive.
-
- * If the value of this field is `@id` and `identity` is false or
- unspecified, the parent field must be resolved using the link
- resolution rules. If `identity` is true, the parent field must be
- resolved using the identifier expansion rules.
-
- * If the value of this field is `@vocab`, the parent field must be
- resolved using the vocabulary resolution rules.
-
- - name: _container
- type: string?
- doc: |
- Structure hint, corresponds to JSON-LD `@container` directive.
- - name: identity
- type: boolean?
- doc: |
- If true and `_type` is `@id` this indicates that the parent field must
- be resolved according to identity resolution rules instead of link
- resolution rules. In addition, the field value is considered an
- assertion that the linked value exists; absence of an object in the loaded document
- with the URI is not an error.
- - name: noLinkCheck
- type: boolean?
- doc: |
- If true, this indicates that link validation traversal must stop at
- this field. This field (it is is a URI) or any fields under it (if it
- is an object or array) are not subject to link checking.
- - name: mapSubject
- type: string?
- doc: |
- If the value of the field is a JSON object, it must be transformed
- into an array of JSON objects, where each key-value pair from the
- source JSON object is a list item, the list items must be JSON objects,
- and the key is assigned to the field specified by `mapSubject`.
- - name: mapPredicate
- type: string?
- doc: |
- Only applies if `mapSubject` is also provided. If the value of the
- field is a JSON object, it is transformed as described in `mapSubject`,
- with the addition that when the value of a map item is not an object,
- the item is transformed to a JSON object with the key assigned to the
- field specified by `mapSubject` and the value assigned to the field
- specified by `mapPredicate`.
- - name: refScope
- type: int?
- doc: |
- If the field contains a relative reference, it must be resolved by
- searching for valid document references in each successive parent scope
- in the document fragment. For example, a reference of `foo` in the
- context `#foo/bar/baz` will first check for the existence of
- `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and
- then finally `#foo`. The first valid URI in the search order shall be
- used as the fully resolved value of the identifier. The value of the
- refScope field is the specified number of levels from the containing
- identifer scope before starting the search, so if `refScope: 2` then
- "baz" and "bar" must be stripped to get the base `#foo` and search
- `#foo/foo` and the `#foo`. The last scope searched must be the top
- level scope before determining if the identifier cannot be resolved.
- - name: typeDSL
- type: boolean?
- doc: |
- Field must be expanded based on the the Schema Salad type DSL.
-
-
-- name: SpecializeDef
- type: record
- fields:
- - name: specializeFrom
- type: string
- doc: "The data type to be replaced"
- jsonldPredicate:
- _id: "sld:specializeFrom"
- _type: "@id"
- refScope: 1
-
- - name: specializeTo
- type: string
- doc: "The new data type to replace with"
- jsonldPredicate:
- _id: "sld:specializeTo"
- _type: "@id"
- refScope: 1
-
-
-- name: NamedType
- type: record
- abstract: true
- fields:
- - name: name
- type: string
- jsonldPredicate: "@id"
- doc: "The identifier for this type"
-
-
-- name: DocType
- type: record
- abstract: true
- fields:
- - name: doc
- type:
- - string?
- - string[]?
- doc: "A documentation string for this type, or an array of strings which should be concatenated."
- jsonldPredicate: "rdfs:comment"
-
- - name: docParent
- type: string?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for this type should appear in a subsection under `docParent`.
- jsonldPredicate:
- _id: "sld:docParent"
- _type: "@id"
-
- - name: docChild
- type:
- - string?
- - string[]?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for `docChild` should appear in a subsection under this type.
- jsonldPredicate:
- _id: "sld:docChild"
- _type: "@id"
-
- - name: docAfter
- type: string?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for this type should appear after the `docAfter` section at the same
- level.
- jsonldPredicate:
- _id: "sld:docAfter"
- _type: "@id"
-
-
-- name: SchemaDefinedType
- type: record
- extends: DocType
- doc: |
- Abstract base for schema-defined types.
- abstract: true
- fields:
- - name: jsonldPredicate
- type:
- - string?
- - JsonldPredicate?
- doc: |
- Annotate this type with linked data context.
- jsonldPredicate: sld:jsonldPredicate
-
- - name: documentRoot
- type: boolean?
- doc: |
- If true, indicates that the type is a valid at the document root. At
- least one type in a schema must be tagged with `documentRoot: true`.
-
-
-- name: SaladRecordField
- type: record
- extends: RecordField
- doc: "A field of a record."
- fields:
- - name: jsonldPredicate
- type:
- - string?
- - JsonldPredicate?
- doc: |
- Annotate this type with linked data context.
- jsonldPredicate: "sld:jsonldPredicate"
-
-
-- name: SaladRecordSchema
- type: record
- extends: [NamedType, RecordSchema, SchemaDefinedType]
- documentRoot: true
- specialize:
- RecordField: SaladRecordField
- fields:
- - name: abstract
- type: boolean?
- doc: |
- If true, this record is abstract and may be used as a base for other
- records, but is not valid on its own.
-
- - name: extends
- type:
- - string?
- - string[]?
- jsonldPredicate:
- _id: "sld:extends"
- _type: "@id"
- refScope: 1
- doc: |
- Indicates that this record inherits fields from one or more base records.
-
- - name: specialize
- type:
- - SpecializeDef[]?
- doc: |
- Only applies if `extends` is declared. Apply type specialization using the
- base record as a template. For each field inherited from the base
- record, replace any instance of the type `specializeFrom` with
- `specializeTo`.
- jsonldPredicate:
- _id: "sld:specialize"
- mapSubject: specializeFrom
- mapPredicate: specializeTo
-
-- name: SaladEnumSchema
- type: record
- extends: [EnumSchema, SchemaDefinedType]
- documentRoot: true
- doc: |
- Define an enumerated type.
- fields:
- - name: extends
- type:
- - string?
- - string[]?
- jsonldPredicate:
- _id: "sld:extends"
- _type: "@id"
- refScope: 1
- doc: |
- Indicates that this enum inherits symbols from a base enum.
-
-
-- name: Documentation
- type: record
- extends: [NamedType, DocType]
- documentRoot: true
- doc: |
- A documentation section. This type exists to facilitate self-documenting
- schemas but has no role in formal validation.
- fields:
- type:
- doc: {foo: "Must be `documentation`"}
- type:
- name: Documentation_symbol
- type: enum
- symbols:
- - "sld:documentation"
- jsonldPredicate:
- _id: "sld:type"
- _type: "@vocab"
- typeDSL: true
- refScope: 2
diff --git a/schema_salad/metaschema/metaschema2.yml~ b/schema_salad/metaschema/metaschema2.yml~
deleted file mode 100644
index 9eed501..0000000
--- a/schema_salad/metaschema/metaschema2.yml~
+++ /dev/null
@@ -1,318 +0,0 @@
-$base: "https://w3id.org/cwl/salad#"
-
-$namespaces:
- sld: "https://w3id.org/cwl/salad#"
- dct: "http://purl.org/dc/terms/"
- rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
- rdfs: "http://www.w3.org/2000/01/rdf-schema#"
- xsd: "http://www.w3.org/2001/XMLSchema#"
-
-$graph:
-
-- name: "Semantic_Annotations_for_Linked_Avro_Data"
- type: documentation
- doc:
- - $include: salad.md
- - $import: field_name.yml
- - $import: ident_res.yml
- - $import: link_res.yml
- - $import: vocab_res.yml
- - $include: import_include.md
-
-- name: "Link_Validation"
- type: documentation
- doc: |
- # Link validation
-
- Once a document has been preprocessed, an implementation may validate
- links. The link validation traversal may visit fields which the schema
- designates as link fields and check that each URI references an existing
- object in the current document, an imported document, file system, or
- network resource. Failure to validate links may be a fatal error. Link
- validation behavior for individual fields may be modified by `identity` and
- `noLinkCheck` in the `jsonldPredicate` section of the field schema.
-
-
-- name: "Schema_validation"
- type: documentation
- doc: ""
-
-
-# - name: "JSON_LD_Context"
-# type: documentation
-# doc: |
-# # Generating JSON-LD Context
-
-# How to generate the json-ld context...
-
-
-- $import: metaschema_base.yml
-
-- name: JsonldPredicate
- type: record
- doc: |
- Attached to a record field to define how the parent record field is handled for
- URI resolution and JSON-LD context generation.
- fields:
- - name: _id
- type: string?
- jsonldPredicate:
- _id: sld:_id
- _type: "@id"
- identity: true
- doc: |
- The predicate URI that this field corresponds to.
- Corresponds to JSON-LD `@id` directive.
- - name: _type
- type: string?
- doc: |
- The context type hint, corresponds to JSON-LD `@type` directive.
-
- * If the value of this field is `@id` and `identity` is false or
- unspecified, the parent field must be resolved using the link
- resolution rules. If `identity` is true, the parent field must be
- resolved using the identifier expansion rules.
-
- * If the value of this field is `@vocab`, the parent field must be
- resolved using the vocabulary resolution rules.
-
- - name: _container
- type: string?
- doc: |
- Structure hint, corresponds to JSON-LD `@container` directive.
- - name: identity
- type: boolean?
- doc: |
- If true and `_type` is `@id` this indicates that the parent field must
- be resolved according to identity resolution rules instead of link
- resolution rules. In addition, the field value is considered an
- assertion that the linked value exists; absence of an object in the loaded document
- with the URI is not an error.
- - name: noLinkCheck
- type: boolean?
- doc: |
- If true, this indicates that link validation traversal must stop at
- this field. This field (it is is a URI) or any fields under it (if it
- is an object or array) are not subject to link checking.
- - name: mapSubject
- type: string?
- doc: |
- If the value of the field is a JSON object, it must be transformed
- into an array of JSON objects, where each key-value pair from the
- source JSON object is a list item, the list items must be JSON objects,
- and the key is assigned to the field specified by `mapSubject`.
- - name: mapPredicate
- type: string?
- doc: |
- Only applies if `mapSubject` is also provided. If the value of the
- field is a JSON object, it is transformed as described in `mapSubject`,
- with the addition that when the value of a map item is not an object,
- the item is transformed to a JSON object with the key assigned to the
- field specified by `mapSubject` and the value assigned to the field
- specified by `mapPredicate`.
- - name: refScope
- type: int?
- doc: |
- If the field contains a relative reference, it must be resolved by
- searching for valid document references in each successive parent scope
- in the document fragment. For example, a reference of `foo` in the
- context `#foo/bar/baz` will first check for the existence of
- `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and
- then finally `#foo`. The first valid URI in the search order shall be
- used as the fully resolved value of the identifier. The value of the
- refScope field is the specified number of levels from the containing
- identifer scope before starting the search, so if `refScope: 2` then
- "baz" and "bar" must be stripped to get the base `#foo` and search
- `#foo/foo` and the `#foo`. The last scope searched must be the top
- level scope before determining if the identifier cannot be resolved.
- - name: typeDSL
- type: boolean?
- doc: |
- Field must be expanded based on the the Schema Salad type DSL.
-
-
-- name: SpecializeDef
- type: record
- fields:
- - name: specializeFrom
- type: string
- doc: "The data type to be replaced"
- jsonldPredicate:
- _id: "sld:specializeFrom"
- _type: "@id"
- refScope: 1
-
- - name: specializeTo
- type: string
- doc: "The new data type to replace with"
- jsonldPredicate:
- _id: "sld:specializeTo"
- _type: "@id"
- refScope: 1
-
-
-- name: NamedType
- type: record
- abstract: true
- fields:
- - name: name
- type: string
- jsonldPredicate: "@id"
- doc: "The identifier for this type"
-
-
-- name: DocType
- type: record
- abstract: true
- fields:
- - name: doc
- type:
- - string?
- - string[]?
- doc: "A documentation string for this type, or an array of strings which should be concatenated."
- jsonldPredicate: "rdfs:comment"
-
- - name: docParent
- type: string?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for this type should appear in a subsection under `docParent`.
- jsonldPredicate:
- _id: "sld:docParent"
- _type: "@id"
-
- - name: docChild
- type:
- - string?
- - string[]?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for `docChild` should appear in a subsection under this type.
- jsonldPredicate:
- _id: "sld:docChild"
- _type: "@id"
-
- - name: docAfter
- type: string?
- doc: |
- Hint to indicate that during documentation generation, documentation
- for this type should appear after the `docAfter` section at the same
- level.
- jsonldPredicate:
- _id: "sld:docAfter"
- _type: "@id"
-
-
-- name: SchemaDefinedType
- type: record
- extends: DocType
- doc: |
- Abstract base for schema-defined types.
- abstract: true
- fields:
- - name: jsonldPredicate
- type:
- - string?
- - JsonldPredicate?
- doc: |
- Annotate this type with linked data context.
- jsonldPredicate: sld:jsonldPredicate
-
- - name: documentRoot
- type: boolean?
- doc: |
- If true, indicates that the type is a valid at the document root. At
- least one type in a schema must be tagged with `documentRoot: true`.
-
-
-- name: SaladRecordField
- type: record
- extends: RecordField
- doc: "A field of a record."
- fields:
- - name: jsonldPredicate
- type:
- - string?
- - JsonldPredicate?
- doc: |
- Annotate this type with linked data context.
- jsonldPredicate: "sld:jsonldPredicate"
-
-
-- name: SaladRecordSchema
- type: record
- extends: [NamedType, RecordSchema, SchemaDefinedType]
- documentRoot: true
- specialize:
- RecordField: SaladRecordField
- fields:
- - name: abstract
- type: boolean?
- doc: |
- If true, this record is abstract and may be used as a base for other
- records, but is not valid on its own.
-
- - name: extends
- type:
- - string?
- - string[]?
- jsonldPredicate:
- _id: "sld:extends"
- _type: "@id"
- refScope: 1
- doc: |
- Indicates that this record inherits fields from one or more base records.
-
- - name: specialize
- type:
- - SpecializeDef[]?
- doc: |
- Only applies if `extends` is declared. Apply type specialization using the
- base record as a template. For each field inherited from the base
- record, replace any instance of the type `specializeFrom` with
- `specializeTo`.
- jsonldPredicate:
- _id: "sld:specialize"
- mapSubject: specializeFrom
- mapPredicate: specializeTo
-
-- name: SaladEnumSchema
- type: record
- extends: [EnumSchema, SchemaDefinedType]
- documentRoot: true
- doc: |
- Define an enumerated type.
- fields:
- - name: extends
- type:
- - string?
- - string[]?
- jsonldPredicate:
- _id: "sld:extends"
- _type: "@id"
- refScope: 1
- doc: |
- Indicates that this enum inherits symbols from a base enum.
-
-
-- name: Documentation
- type: record
- extends: [NamedType, DocType]
- documentRoot: true
- doc: |
- A documentation section. This type exists to facilitate self-documenting
- schemas but has no role in formal validation.
- fields:
- bloop: blip
- type:
- doc: "Must be `documentation`"
- type:
- name: Documentation_symbol
- type: enum
- symbols:
- - "sld:documentation"
- jsonldPredicate:
- _id: "sld:type"
- _type: "@vocab"
- typeDSL: true
- refScope: 2
diff --git a/schema_salad/metaschema/metaschema_base.yml b/schema_salad/metaschema/metaschema_base.yml
index 73511d1..d8bf0a3 100644
--- a/schema_salad/metaschema/metaschema_base.yml
+++ b/schema_salad/metaschema/metaschema_base.yml
@@ -8,6 +8,12 @@ $namespaces:
xsd: "http://www.w3.org/2001/XMLSchema#"
$graph:
+
+- name: "Schema"
+ type: documentation
+ doc: |
+ # Schema
+
- name: PrimitiveType
type: enum
symbols:
@@ -35,6 +41,7 @@ $graph:
- name: Any
type: enum
symbols: ["#Any"]
+ docAfter: "#PrimitiveType"
doc: |
The **Any** type validates for any non-null value.
diff --git a/schema_salad/metaschema/salad.md b/schema_salad/metaschema/salad.md
index 6dd3e6a..2d4681e 100644
--- a/schema_salad/metaschema/salad.md
+++ b/schema_salad/metaschema/salad.md
@@ -26,7 +26,7 @@ Web.
This document is the product of the [Common Workflow Language working
group](https://groups.google.com/forum/#!forum/common-workflow-language). The
-latest version of this document is available in the "schema_salad" directory at
+latest version of this document is available in the "schema_salad" repository at
https://github.com/common-workflow-language/schema_salad
@@ -38,7 +38,7 @@ under the terms of the Apache License, version 2.0.
# Introduction
The JSON data model is an extremely popular way to represent structured
-data. It is attractive because of it's relative simplicity and is a
+data. It is attractive because of its relative simplicity and is a
natural fit with the standard types of many programming languages.
However, this simplicity means that basic JSON lacks expressive features
useful for working with complex data structures and document formats, such
@@ -70,12 +70,17 @@ and RDF schema, and production of RDF triples by applying the JSON-LD
context. The schema language also provides for robust support of inline
documentation.
-## Introduction to draft 1
+## Introduction to v1.0
-This is the first version of Schema Salad. It is developed concurrently
-with draft 3 of the Common Workflow Language for use in specifying the
-Common Workflow Language, however Schema Salad is intended to be useful to
-a broader audience.
+This is the second version of of the Schema Salad specification. It is
+developed concurrently with v1.0 of the Common Workflow Language for use in
+specifying the Common Workflow Language, however Schema Salad is intended to be
+useful to a broader audience. Compared to the draft-1 schema salad
+specification, the following changes have been made:
+
+* Use of [mapSubject and mapPredicate](#Identifier_maps) to transform maps to lists of records.
+* Resolution of the [domain Specific Language for types](#Domain_Specific_Language_for_types)
+* Consolidation of the formal [schema into section 5](#Schema).
## References to Other Specifications
diff --git a/schema_salad/metaschema/typedsl_res.yml b/schema_salad/metaschema/typedsl_res.yml
new file mode 100644
index 0000000..b1a0c1d
--- /dev/null
+++ b/schema_salad/metaschema/typedsl_res.yml
@@ -0,0 +1,33 @@
+- |
+ ## Domain Specific Language for types
+
+ Fields may be tagged `typeDSL: true`. If so, the field is expanded using the
+ following micro-DSL for schema salad types:
+
+ * If the type ends with a question mark `?` it is expanded to a union with `null`
+ * If the type ends with square brackets `[]` it is expanded to an array with items of the preceeding type symbol
+ * The type may end with both `[]?` to indicate it is an optional array.
+ * Identifier resolution is applied after type DSL expansion.
+
+ ### Type DSL example
+
+ Given the following schema:
+
+ ```
+- $include: typedsl_res_schema.yml
+- |
+ ```
+
+ Process the following example:
+
+ ```
+- $include: typedsl_res_src.yml
+- |
+ ```
+
+ This becomes:
+
+ ```
+- $include: typedsl_res_proc.yml
+- |
+ ```
diff --git a/schema_salad/metaschema/typedsl_res_proc.yml b/schema_salad/metaschema/typedsl_res_proc.yml
new file mode 100644
index 0000000..8097a6a
--- /dev/null
+++ b/schema_salad/metaschema/typedsl_res_proc.yml
@@ -0,0 +1,26 @@
+[
+ {
+ "extype": "string"
+ },
+ {
+ "extype": [
+ "null",
+ "string"
+ ]
+ },
+ {
+ "extype": {
+ "type": "array",
+ "items": "string"
+ }
+ },
+ {
+ "extype": [
+ "null",
+ {
+ "type": "array",
+ "items": "string"
+ }
+ ]
+ }
+]
diff --git a/schema_salad/metaschema/typedsl_res_schema.yml b/schema_salad/metaschema/typedsl_res_schema.yml
new file mode 100644
index 0000000..52459a6
--- /dev/null
+++ b/schema_salad/metaschema/typedsl_res_schema.yml
@@ -0,0 +1,17 @@
+{
+ "$graph": [
+ {"$import": "metaschema_base.yml"},
+ {
+ "name": "TypeDSLExample",
+ "type": "record",
+ "documentRoot": true,
+ "fields": [{
+ "name": "extype",
+ "type": "string",
+ "jsonldPredicate": {
+ _type: "@vocab",
+ "typeDSL": true
+ }
+ }]
+ }]
+}
diff --git a/schema_salad/metaschema/typedsl_res_src.yml b/schema_salad/metaschema/typedsl_res_src.yml
new file mode 100644
index 0000000..6ecbd50
--- /dev/null
+++ b/schema_salad/metaschema/typedsl_res_src.yml
@@ -0,0 +1,9 @@
+[{
+ "extype": "string"
+}, {
+ "extype": "string?"
+}, {
+ "extype": "string[]"
+}, {
+ "extype": "string[]?"
+}]
diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py
index b1f64c8..d5d1408 100644
--- a/schema_salad/ref_resolver.py
+++ b/schema_salad/ref_resolver.py
@@ -1,19 +1,22 @@
+from __future__ import absolute_import
import sys
import os
import json
import hashlib
import logging
import collections
-import urllib
-import urlparse
+
+import six
+from six.moves import range
+from six.moves import urllib
+from six import StringIO
+
import re
import copy
-import urllib
-from StringIO import StringIO
+
from . import validate
-from .aslist import aslist
-from .flatten import flatten
+from .utils import aslist, flatten
from .sourceline import SourceLine, add_lc_filename, relname
import requests
@@ -27,53 +30,60 @@ from rdflib import Graph
from rdflib.namespace import RDF, RDFS, OWL
from rdflib.plugins.parsers.notation3 import BadSyntax
import xml.sax
-from typing import (Any, AnyStr, Callable, cast, Dict, List, Iterable, Tuple,
- TypeVar, Union)
+from typing import (cast, Any, AnyStr, Callable, Dict, List, Iterable,
+ Optional, Set, Text, Tuple, TypeVar, Union)
+
_logger = logging.getLogger("salad")
-ContextType = Dict[unicode, Union[Dict, unicode, Iterable[unicode]]]
+ContextType = Dict[six.text_type, Union[Dict, six.text_type, Iterable[six.text_type]]]
DocumentType = TypeVar('DocumentType', CommentedSeq, CommentedMap)
DocumentOrStrType = TypeVar(
- 'DocumentOrStrType', CommentedSeq, CommentedMap, unicode)
+ 'DocumentOrStrType', CommentedSeq, CommentedMap, six.text_type)
-def file_uri(path): # type: (str) -> str
+def file_uri(path, split_frag=False): # type: (str, bool) -> str
if path.startswith("file://"):
return path
- pathsp = path.split("#", 2)
- frag = "#" + urllib.quote(str(pathsp[1])) if len(pathsp) == 2 else ""
- urlpath = urllib.pathname2url(str(pathsp[0]))
+ if split_frag:
+ pathsp = path.split("#", 2)
+ frag = "#" + urllib.parse.quote(str(pathsp[1])) if len(pathsp) == 2 else ""
+ urlpath = urllib.request.pathname2url(str(pathsp[0]))
+ else:
+ urlpath = urllib.request.pathname2url(path)
+ frag = ""
if urlpath.startswith("//"):
return "file:%s%s" % (urlpath, frag)
else:
return "file://%s%s" % (urlpath, frag)
def uri_file_path(url): # type: (str) -> str
- split = urlparse.urlsplit(url)
+ split = urllib.parse.urlsplit(url)
if split.scheme == "file":
- return urllib.url2pathname(str(split.path)) + ("#" + urllib.unquote(str(split.fragment)) if split.fragment else "")
+ return urllib.request.url2pathname(
+ str(split.path)) + ("#" + urllib.parse.unquote(str(split.fragment))
+ if bool(split.fragment) else "")
else:
raise ValueError("Not a file URI")
class NormDict(CommentedMap):
- def __init__(self, normalize=unicode): # type: (type) -> None
+ def __init__(self, normalize=six.text_type): # type: (Callable) -> None
super(NormDict, self).__init__()
self.normalize = normalize
- def __getitem__(self, key):
+ def __getitem__(self, key): # type: (Any) -> Any
return super(NormDict, self).__getitem__(self.normalize(key))
- def __setitem__(self, key, value):
+ def __setitem__(self, key, value): # type: (Any, Any) -> Any
return super(NormDict, self).__setitem__(self.normalize(key), value)
- def __delitem__(self, key):
+ def __delitem__(self, key): # type: (Any) -> Any
return super(NormDict, self).__delitem__(self.normalize(key))
- def __contains__(self, key):
+ def __contains__(self, key): # type: (Any) -> Any
return super(NormDict, self).__contains__(self.normalize(key))
-def merge_properties(a, b):
+def merge_properties(a, b): # type: (List[Any], List[Any]) -> Dict[Any, Any]
c = {}
for i in a:
if i not in b:
@@ -91,33 +101,37 @@ def merge_properties(a, b):
def SubLoader(loader): # type: (Loader) -> Loader
return Loader(loader.ctx, schemagraph=loader.graph,
foreign_properties=loader.foreign_properties, idx=loader.idx,
- cache=loader.cache, fetcher_constructor=loader.fetcher_constructor)
+ cache=loader.cache, fetcher_constructor=loader.fetcher_constructor,
+ skip_schemas=loader.skip_schemas)
class Fetcher(object):
- def fetch_text(self, url): # type: (unicode) -> unicode
+ def fetch_text(self, url): # type: (Text) -> Text
raise NotImplementedError()
- def check_exists(self, url): # type: (unicode) -> bool
+ def check_exists(self, url): # type: (Text) -> bool
raise NotImplementedError()
- def urljoin(self, base_url, url): # type: (unicode, unicode) -> unicode
+ def urljoin(self, base_url, url): # type: (Text, Text) -> Text
raise NotImplementedError()
class DefaultFetcher(Fetcher):
- def __init__(self, cache, session): # type: (dict, requests.sessions.Session) -> None
+ def __init__(self,
+ cache, # type: Dict[Text, Text]
+ session # type: Optional[requests.sessions.Session]
+ ): # type: (...) -> None
self.cache = cache
self.session = session
def fetch_text(self, url):
- # type: (unicode) -> unicode
+ # type: (Text) -> Text
if url in self.cache:
return self.cache[url]
- split = urlparse.urlsplit(url)
+ split = urllib.parse.urlsplit(url)
scheme, path = split.scheme, split.path
- if scheme in [u'http', u'https'] and self.session:
+ if scheme in [u'http', u'https'] and self.session is not None:
try:
resp = self.session.get(url)
resp.raise_for_status()
@@ -126,7 +140,7 @@ class DefaultFetcher(Fetcher):
return resp.text
elif scheme == 'file':
try:
- with open(urllib.url2pathname(str(path))) as fp:
+ with open(urllib.request.url2pathname(str(path))) as fp:
read = fp.read()
if hasattr(read, "decode"):
return read.decode("utf-8")
@@ -134,20 +148,20 @@ class DefaultFetcher(Fetcher):
return read
except (OSError, IOError) as e:
if e.filename == path:
- raise RuntimeError(unicode(e))
+ raise RuntimeError(six.text_type(e))
else:
raise RuntimeError('Error reading %s: %s' % (url, e))
else:
raise ValueError('Unsupported scheme in url: %s' % url)
- def check_exists(self, url): # type: (unicode) -> bool
+ def check_exists(self, url): # type: (Text) -> bool
if url in self.cache:
return True
- split = urlparse.urlsplit(url)
+ split = urllib.parse.urlsplit(url)
scheme, path = split.scheme, split.path
- if scheme in [u'http', u'https'] and self.session:
+ if scheme in [u'http', u'https'] and self.session is not None:
try:
resp = self.session.head(url)
resp.raise_for_status()
@@ -155,58 +169,70 @@ class DefaultFetcher(Fetcher):
return False
return True
elif scheme == 'file':
- return os.path.exists(urllib.url2pathname(str(path)))
+ return os.path.exists(urllib.request.url2pathname(str(path)))
else:
raise ValueError('Unsupported scheme in url: %s' % url)
- def urljoin(self, base_url, url):
- return urlparse.urljoin(base_url, url)
+ def urljoin(self, base_url, url): # type: (Text, Text) -> Text
+ return urllib.parse.urljoin(base_url, url)
class Loader(object):
def __init__(self,
ctx, # type: ContextType
schemagraph=None, # type: rdflib.graph.Graph
- foreign_properties=None, # type: Set[unicode]
- idx=None, # type: Dict[unicode, Union[CommentedMap, CommentedSeq, unicode]]
- cache=None, # type: Dict[unicode, Any]
+ foreign_properties=None, # type: Set[Text]
+ idx=None, # type: Dict[Text, Union[CommentedMap, CommentedSeq, Text, None]]
+ cache=None, # type: Dict[Text, Any]
session=None, # type: requests.sessions.Session
- fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
+ fetcher_constructor=None, # type: Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]
+ skip_schemas=None # type: bool
):
# type: (...) -> None
- normalize = lambda url: urlparse.urlsplit(url).geturl()
- self.idx = None # type: Dict[unicode, Union[CommentedMap, CommentedSeq, unicode]]
+ normalize = lambda url: urllib.parse.urlsplit(url).geturl()
if idx is not None:
self.idx = idx
else:
self.idx = NormDict(normalize)
self.ctx = {} # type: ContextType
- self.graph = None # type: Graph
if schemagraph is not None:
self.graph = schemagraph
else:
self.graph = rdflib.graph.Graph()
- self.foreign_properties = None # type: Set[unicode]
if foreign_properties is not None:
self.foreign_properties = foreign_properties
else:
self.foreign_properties = set()
- self.cache = None # type: Dict[unicode, Any]
if cache is not None:
self.cache = cache
else:
self.cache = {}
+ if skip_schemas is not None:
+ self.skip_schemas = skip_schemas
+ else:
+ self.skip_schemas = False
+
if session is None:
- self.session = CacheControl(requests.Session(),
- cache=FileCache(os.path.join(os.environ["HOME"], ".cache", "salad")))
+ if "HOME" in os.environ:
+ self.session = CacheControl(
+ requests.Session(),
+ cache=FileCache(os.path.join(os.environ["HOME"], ".cache", "salad")))
+ elif "TMP" in os.environ:
+ self.session = CacheControl(
+ requests.Session(),
+ cache=FileCache(os.path.join(os.environ["TMP"], ".cache", "salad")))
+ else:
+ self.session = CacheControl(
+ requests.Session(),
+ cache=FileCache("/tmp", ".cache", "salad"))
else:
self.session = session
- if fetcher_constructor:
+ if fetcher_constructor is not None:
self.fetcher_constructor = fetcher_constructor
else:
self.fetcher_constructor = DefaultFetcher
@@ -215,53 +241,54 @@ class Loader(object):
self.fetch_text = self.fetcher.fetch_text
self.check_exists = self.fetcher.check_exists
- self.url_fields = None # type: Set[unicode]
- self.scoped_ref_fields = None # type: Dict[unicode, int]
- self.vocab_fields = None # type: Set[unicode]
- self.identifiers = None # type: Set[unicode]
- self.identity_links = None # type: Set[unicode]
- self.standalone = None # type: Set[unicode]
- self.nolinkcheck = None # type: Set[unicode]
- self.vocab = {} # type: Dict[unicode, unicode]
- self.rvocab = {} # type: Dict[unicode, unicode]
- self.idmap = None # type: Dict[unicode, Any]
- self.mapPredicate = None # type: Dict[unicode, unicode]
- self.type_dsl_fields = None # type: Set[unicode]
+ self.url_fields = set() # type: Set[Text]
+ self.scoped_ref_fields = {} # type: Dict[Text, int]
+ self.vocab_fields = set() # type: Set[Text]
+ self.identifiers = [] # type: List[Text]
+ self.identity_links = set() # type: Set[Text]
+ self.standalone = None # type: Optional[Set[Text]]
+ self.nolinkcheck = set() # type: Set[Text]
+ self.vocab = {} # type: Dict[Text, Text]
+ self.rvocab = {} # type: Dict[Text, Text]
+ self.idmap = {} # type: Dict[Text, Any]
+ self.mapPredicate = {} # type: Dict[Text, Text]
+ self.type_dsl_fields = set() # type: Set[Text]
self.add_context(ctx)
def expand_url(self,
- url, # type: unicode
- base_url, # type: unicode
+ url, # type: Text
+ base_url, # type: Text
scoped_id=False, # type: bool
vocab_term=False, # type: bool
scoped_ref=None # type: int
):
- # type: (...) -> unicode
+ # type: (...) -> Text
if url in (u"@id", u"@type"):
return url
if vocab_term and url in self.vocab:
return url
- if self.vocab and u":" in url:
+ if bool(self.vocab) and u":" in url:
prefix = url.split(u":")[0]
if prefix in self.vocab:
url = self.vocab[prefix] + url[len(prefix) + 1:]
- split = urlparse.urlsplit(url)
+ split = urllib.parse.urlsplit(url)
- if split.scheme or url.startswith(u"$(") or url.startswith(u"${"):
+ if (bool(split.scheme) or url.startswith(u"$(")
+ or url.startswith(u"${")):
pass
- elif scoped_id and not split.fragment:
- splitbase = urlparse.urlsplit(base_url)
+ elif scoped_id and not bool(split.fragment):
+ splitbase = urllib.parse.urlsplit(base_url)
frg = u""
- if splitbase.fragment:
+ if bool(splitbase.fragment):
frg = splitbase.fragment + u"/" + split.path
else:
frg = split.path
- pt = splitbase.path if splitbase.path else "/"
- url = urlparse.urlunsplit(
+ pt = splitbase.path if splitbase.path != '' else "/"
+ url = urllib.parse.urlunsplit(
(splitbase.scheme, splitbase.netloc, pt, splitbase.query, frg))
elif scoped_ref is not None and not split.fragment:
pass
@@ -273,22 +300,24 @@ class Loader(object):
else:
return url
- def _add_properties(self, s): # type: (unicode) -> None
+ def _add_properties(self, s): # type: (Text) -> None
for _, _, rng in self.graph.triples((s, RDFS.range, None)):
- literal = ((unicode(rng).startswith(
+ literal = ((six.text_type(rng).startswith(
u"http://www.w3.org/2001/XMLSchema#") and
- not unicode(rng) == u"http://www.w3.org/2001/XMLSchema#anyURI")
- or unicode(rng) ==
+ not six.text_type(rng) == u"http://www.w3.org/2001/XMLSchema#anyURI")
+ or six.text_type(rng) ==
u"http://www.w3.org/2000/01/rdf-schema#Literal")
if not literal:
- self.url_fields.add(unicode(s))
- self.foreign_properties.add(unicode(s))
+ self.url_fields.add(six.text_type(s))
+ self.foreign_properties.add(six.text_type(s))
- def add_namespaces(self, ns): # type: (Dict[unicode, unicode]) -> None
+ def add_namespaces(self, ns): # type: (Dict[Text, Text]) -> None
self.vocab.update(ns)
def add_schemas(self, ns, base_url):
- # type: (Union[List[unicode], unicode], unicode) -> None
+ # type: (Union[List[Text], Text], Text) -> None
+ if self.skip_schemas:
+ return
for sch in aslist(ns):
fetchurl = self.fetcher.urljoin(base_url, sch)
if fetchurl not in self.cache:
@@ -297,7 +326,7 @@ class Loader(object):
self.cache[fetchurl] = rdflib.graph.Graph()
for fmt in ['xml', 'turtle', 'rdfa']:
try:
- self.cache[fetchurl].parse(data=content, format=fmt)
+ self.cache[fetchurl].parse(data=content, format=fmt, publicID=str(fetchurl))
self.graph += self.cache[fetchurl]
break
except xml.sax.SAXParseException:
@@ -318,18 +347,18 @@ class Loader(object):
self._add_properties(s)
for s, _, _ in self.graph.triples((None, None, None)):
- self.idx[unicode(s)] = None
+ self.idx[six.text_type(s)] = None
def add_context(self, newcontext, baseuri=""):
- # type: (ContextType, unicode) -> None
- if self.vocab:
+ # type: (ContextType, Text) -> None
+ if bool(self.vocab):
raise validate.ValidationException(
"Refreshing context that already has stuff in it")
self.url_fields = set(("$schemas",))
self.scoped_ref_fields = {}
self.vocab_fields = set()
- self.identifiers = set()
+ self.identifiers = []
self.identity_links = set()
self.standalone = set()
self.nolinkcheck = set()
@@ -345,7 +374,7 @@ class Loader(object):
for key, value in self.ctx.items():
if value == u"@id":
- self.identifiers.add(key)
+ self.identifiers.append(key)
self.identity_links.add(key)
elif isinstance(value, dict) and value.get(u"@type") == u"@id":
self.url_fields.add(key)
@@ -371,12 +400,14 @@ class Loader(object):
if isinstance(value, dict) and u"@id" in value:
self.vocab[key] = value[u"@id"]
- elif isinstance(value, basestring):
+ elif isinstance(value, six.string_types):
self.vocab[key] = value
for k, v in self.vocab.items():
self.rvocab[self.expand_url(v, u"", scoped_id=False)] = k
+ self.identifiers.sort()
+
_logger.debug("identifiers is %s", self.identifiers)
_logger.debug("identity_links is %s", self.identity_links)
_logger.debug("url_fields is %s", self.url_fields)
@@ -384,64 +415,69 @@ class Loader(object):
_logger.debug("vocab is %s", self.vocab)
def resolve_ref(self,
- ref, # type: Union[CommentedMap, CommentedSeq, unicode]
- base_url=None, # type: unicode
+ ref, # type: Union[CommentedMap, CommentedSeq, Text]
+ base_url=None, # type: Text
checklinks=True # type: bool
):
- # type: (...) -> Tuple[Union[CommentedMap, CommentedSeq, unicode], Dict[unicode, Any]]
+ # type: (...) -> Tuple[Union[CommentedMap, CommentedSeq, Text, None], Dict[Text, Any]]
- obj = None # type: CommentedMap
- resolved_obj = None # type: Union[CommentedMap, CommentedSeq, unicode]
+ lref = ref # type: Union[CommentedMap, CommentedSeq, Text, None]
+ obj = None # type: Optional[CommentedMap]
+ resolved_obj = None # type: Optional[Union[CommentedMap, CommentedSeq, Text]]
inc = False
- mixin = None # type: Dict[unicode, Any]
+ mixin = None # type: Optional[Dict[Text, Any]]
if not base_url:
base_url = file_uri(os.getcwd()) + "/"
- if isinstance(ref, (str, unicode)) and os.sep == "\\":
+ if isinstance(lref, (str, six.text_type)) and os.sep == "\\":
# Convert Windows path separator in ref
- ref = ref.replace("\\", "/")
+ lref = lref.replace("\\", "/")
sl = SourceLine(obj, None, ValueError)
# If `ref` is a dict, look for special directives.
- if isinstance(ref, CommentedMap):
- obj = ref
+ if isinstance(lref, CommentedMap):
+ obj = lref
if "$import" in obj:
sl = SourceLine(obj, "$import", RuntimeError)
if len(obj) == 1:
- ref = obj[u"$import"]
+ lref = obj[u"$import"]
obj = None
else:
raise sl.makeError(
- u"'$import' must be the only field in %s" % (unicode(obj)))
+ u"'$import' must be the only field in %s"
+ % (six.text_type(obj)))
elif "$include" in obj:
sl = SourceLine(obj, "$include", RuntimeError)
if len(obj) == 1:
- ref = obj[u"$include"]
+ lref = obj[u"$include"]
inc = True
obj = None
else:
raise sl.makeError(
- u"'$include' must be the only field in %s" % (unicode(obj)))
+ u"'$include' must be the only field in %s"
+ % (six.text_type(obj)))
elif "$mixin" in obj:
sl = SourceLine(obj, "$mixin", RuntimeError)
- ref = obj[u"$mixin"]
+ lref = obj[u"$mixin"]
mixin = obj
obj = None
else:
- ref = None
+ lref = None
for identifier in self.identifiers:
if identifier in obj:
- ref = obj[identifier]
+ lref = obj[identifier]
break
- if not ref:
+ if not lref:
raise sl.makeError(
- u"Object `%s` does not have identifier field in %s" % (relname(obj), self.identifiers))
+ u"Object `%s` does not have identifier field in %s"
+ % (relname(obj), self.identifiers))
- if not isinstance(ref, (str, unicode)):
- raise ValueError(u"Expected CommentedMap or string, got %s: `%s`" % (type(ref), unicode(ref)))
+ if not isinstance(lref, (str, six.text_type)):
+ raise ValueError(u"Expected CommentedMap or string, got %s: `%s`"
+ % (type(lref), six.text_type(lref)))
- url = self.expand_url(ref, base_url, scoped_id=(obj is not None))
+ url = self.expand_url(lref, base_url, scoped_id=(obj is not None))
# Has this reference been loaded already?
if url in self.idx and (not mixin):
return self.idx[url], {}
@@ -453,27 +489,27 @@ class Loader(object):
return self.fetch_text(url), {}
doc = None
- if obj:
+ if isinstance(obj, collections.MutableMapping):
for identifier in self.identifiers:
obj[identifier] = url
doc_url = url
else:
# Load structured document
- doc_url, frg = urlparse.urldefrag(url)
+ doc_url, frg = urllib.parse.urldefrag(url)
if doc_url in self.idx and (not mixin):
# If the base document is in the index, it was already loaded,
# so if we didn't find the reference earlier then it must not
# exist.
raise validate.ValidationException(
- u"Reference `#%s` not found in file `%s`." % (frg, doc_url))
+ u"Reference `#%s` not found in file `%s`."
+ % (frg, doc_url))
doc = self.fetch(doc_url, inject_ids=(not mixin))
# Recursively expand urls and resolve directives
- if mixin:
+ if bool(mixin):
doc = copy.deepcopy(doc)
doc.update(mixin)
del doc["$mixin"]
- url = None
resolved_obj, metadata = self.resolve_all(
doc, base_url, file_base=doc_url, checklinks=checklinks)
else:
@@ -482,7 +518,7 @@ class Loader(object):
# Requested reference should be in the index now, otherwise it's a bad
# reference
- if url is not None:
+ if not bool(mixin):
if url in self.idx:
resolved_obj = self.idx[url]
else:
@@ -515,7 +551,7 @@ class Loader(object):
ls = CommentedSeq()
for k in sorted(idmapFieldValue.keys()):
val = idmapFieldValue[k]
- v = None # type: CommentedMap
+ v = None # type: Optional[CommentedMap]
if not isinstance(val, CommentedMap):
if idmapField in loader.mapPredicate:
v = CommentedMap(
@@ -544,15 +580,15 @@ class Loader(object):
document[idmapField] = ls
- typeDSLregex = re.compile(ur"^([^[?]+)(\[\])?(\?)?$")
+ typeDSLregex = re.compile(u"^([^[?]+)(\[\])?(\?)?$")
def _type_dsl(self,
- t, # type: Union[unicode, Dict, List]
+ t, # type: Union[Text, Dict, List]
lc,
filename):
- # type: (...) -> Union[unicode, Dict[unicode, unicode], List[Union[unicode, Dict[unicode, unicode]]]]
+ # type: (...) -> Union[Text, Dict[Text, Text], List[Union[Text, Dict[Text, Text]]]]
- if not isinstance(t, (str, unicode)):
+ if not isinstance(t, (str, six.text_type)):
return t
m = Loader.typeDSLregex.match(t)
@@ -560,13 +596,13 @@ class Loader(object):
return t
first = m.group(1)
second = third = None
- if m.group(2):
+ if bool(m.group(2)):
second = CommentedMap((("type", "array"),
("items", first)))
second.lc.add_kv_line_col("type", lc)
second.lc.add_kv_line_col("items", lc)
second.lc.filename = filename
- if m.group(3):
+ if bool(m.group(3)):
third = CommentedSeq([u"null", second or first])
third.lc.add_kv_line_col(0, lc)
third.lc.add_kv_line_col(1, lc)
@@ -581,7 +617,7 @@ class Loader(object):
for d in loader.type_dsl_fields:
if d in document:
datum2 = datum = document[d]
- if isinstance(datum, (str, unicode)):
+ if isinstance(datum, (str, six.text_type)):
datum2 = self._type_dsl(datum, document.lc.data[
d], document.lc.filename)
elif isinstance(datum, CommentedSeq):
@@ -593,7 +629,7 @@ class Loader(object):
t, datum.lc.data[n], document.lc.filename))
if isinstance(datum2, CommentedSeq):
datum3 = CommentedSeq()
- seen = [] # type: List[unicode]
+ seen = [] # type: List[Text]
for i, item in enumerate(datum2):
if isinstance(item, CommentedSeq):
for j, v in enumerate(item):
@@ -613,16 +649,16 @@ class Loader(object):
document[d] = datum2
def _resolve_identifier(self, document, loader, base_url):
- # type: (CommentedMap, Loader, unicode) -> unicode
+ # type: (CommentedMap, Loader, Text) -> Text
# Expand identifier field (usually 'id') to resolve scope
for identifer in loader.identifiers:
if identifer in document:
- if isinstance(document[identifer], basestring):
+ if isinstance(document[identifer], six.string_types):
document[identifer] = loader.expand_url(
document[identifer], base_url, scoped_id=True)
if (document[identifer] not in loader.idx
or isinstance(
- loader.idx[document[identifer]], basestring)):
+ loader.idx[document[identifer]], six.string_types)):
loader.idx[document[identifer]] = document
base_url = document[identifer]
else:
@@ -632,13 +668,13 @@ class Loader(object):
return base_url
def _resolve_identity(self, document, loader, base_url):
- # type: (Dict[unicode, List[unicode]], Loader, unicode) -> None
+ # type: (Dict[Text, List[Text]], Loader, Text) -> None
# Resolve scope for identity fields (fields where the value is the
# identity of a standalone node, such as enum symbols)
for identifer in loader.identity_links:
if identifer in document and isinstance(document[identifer], list):
for n, v in enumerate(document[identifer]):
- if isinstance(document[identifer][n], basestring):
+ if isinstance(document[identifer][n], six.string_types):
document[identifer][n] = loader.expand_url(
document[identifer][n], base_url, scoped_id=True)
if document[identifer][n] not in loader.idx:
@@ -646,32 +682,32 @@ class Loader(object):
n]] = document[identifer][n]
def _normalize_fields(self, document, loader):
- # type: (Dict[unicode, unicode], Loader) -> None
+ # type: (Dict[Text, Text], Loader) -> None
# Normalize fields which are prefixed or full URIn to vocabulary terms
- for d in document:
+ for d in list(document.keys()):
d2 = loader.expand_url(d, u"", scoped_id=False, vocab_term=True)
if d != d2:
document[d2] = document[d]
del document[d]
def _resolve_uris(self,
- document, # type: Dict[unicode, Union[unicode, List[unicode]]]
+ document, # type: Dict[Text, Union[Text, List[Text]]]
loader, # type: Loader
- base_url # type: unicode
+ base_url # type: Text
):
# type: (...) -> None
# Resolve remaining URLs based on document base
for d in loader.url_fields:
if d in document:
datum = document[d]
- if isinstance(datum, (str, unicode)):
+ if isinstance(datum, (str, six.text_type)):
document[d] = loader.expand_url(
datum, base_url, scoped_id=False,
vocab_term=(d in loader.vocab_fields),
scoped_ref=self.scoped_ref_fields.get(d))
elif isinstance(datum, list):
for i, url in enumerate(datum):
- if isinstance(url, (str, unicode)):
+ if isinstance(url, (str, six.text_type)):
datum[i] = loader.expand_url(
url, base_url, scoped_id=False,
vocab_term=(d in loader.vocab_fields),
@@ -680,11 +716,11 @@ class Loader(object):
def resolve_all(self,
document, # type: Union[CommentedMap, CommentedSeq]
- base_url, # type: unicode
- file_base=None, # type: unicode
+ base_url, # type: Text
+ file_base=None, # type: Text
checklinks=True # type: bool
):
- # type: (...) -> Tuple[Union[CommentedMap, CommentedSeq, unicode], Dict[unicode, Any]]
+ # type: (...) -> Tuple[Union[CommentedMap, CommentedSeq, Text, None], Dict[Text, Any]]
loader = self
metadata = CommentedMap() # type: CommentedMap
if file_base is None:
@@ -705,14 +741,14 @@ class Loader(object):
else:
return (document, metadata)
- newctx = None # type: Loader
+ newctx = None # type: Optional[Loader]
if isinstance(document, CommentedMap):
# Handle $base, $profile, $namespaces, $schemas and $graph
if u"$base" in document:
base_url = document[u"$base"]
if u"$profile" in document:
- if not newctx:
+ if newctx is None:
newctx = SubLoader(self)
prof = self.fetch(document[u"$profile"])
newctx.add_namespaces(document.get(u"$namespaces", {}))
@@ -720,16 +756,16 @@ class Loader(object):
u"$schemas", []), document[u"$profile"])
if u"$namespaces" in document:
- if not newctx:
+ if newctx is None:
newctx = SubLoader(self)
newctx.add_namespaces(document[u"$namespaces"])
if u"$schemas" in document:
- if not newctx:
+ if newctx is None:
newctx = SubLoader(self)
newctx.add_schemas(document[u"$schemas"], file_base)
- if newctx:
+ if newctx is not None:
loader = newctx
if u"$graph" in document:
@@ -760,7 +796,7 @@ class Loader(object):
except validate.ValidationException as v:
_logger.warn("loader is %s", id(loader), exc_info=True)
raise validate.ValidationException("(%s) (%s) Validation error in field %s:\n%s" % (
- id(loader), file_base, key, validate.indent(unicode(v))))
+ id(loader), file_base, key, validate.indent(six.text_type(v))))
elif isinstance(document, CommentedSeq):
i = 0
@@ -778,7 +814,7 @@ class Loader(object):
document.lc.data[
j - 1] = document.lc.data[j - llen]
for item in l:
- document.insert(i, item) # type: ignore
+ document.insert(i, item)
document.lc.data[i] = lc
i += 1
else:
@@ -791,21 +827,22 @@ class Loader(object):
except validate.ValidationException as v:
_logger.warn("failed", exc_info=True)
raise validate.ValidationException("(%s) (%s) Validation error in position %i:\n%s" % (
- id(loader), file_base, i, validate.indent(unicode(v))))
+ id(loader), file_base, i, validate.indent(six.text_type(v))))
for identifer in loader.identity_links:
if identifer in metadata:
- if isinstance(metadata[identifer], (str, unicode)):
+ if isinstance(metadata[identifer], (str, six.text_type)):
metadata[identifer] = loader.expand_url(
metadata[identifer], base_url, scoped_id=True)
loader.idx[metadata[identifer]] = document
if checklinks:
- self.validate_links(document, u"")
+ all_doc_ids={} # type: Dict[Text, Text]
+ self.validate_links(document, u"", all_doc_ids)
return document, metadata
- def fetch(self, url, inject_ids=True): # type: (unicode, bool) -> Any
+ def fetch(self, url, inject_ids=True): # type: (Text, bool) -> Any
if url in self.idx:
return self.idx[url]
try:
@@ -815,11 +852,12 @@ class Loader(object):
else:
textIO = StringIO(text)
textIO.name = url # type: ignore
- result = yaml.round_trip_load(textIO) # type: ignore
+ result = yaml.round_trip_load(textIO)
add_lc_filename(result, url)
except yaml.parser.ParserError as e:
raise validate.ValidationException("Syntax error %s" % (e))
- if isinstance(result, CommentedMap) and inject_ids and self.identifiers:
+ if (isinstance(result, CommentedMap) and inject_ids
+ and bool(self.identifiers)):
for identifier in self.identifiers:
if identifier not in result:
result[identifier] = url
@@ -829,11 +867,11 @@ class Loader(object):
return result
- FieldType = TypeVar('FieldType', unicode, CommentedSeq, CommentedMap)
+ FieldType = TypeVar('FieldType', six.text_type, CommentedSeq, CommentedMap)
def validate_scoped(self, field, link, docid):
- # type: (unicode, unicode, unicode) -> unicode
- split = urlparse.urlsplit(docid)
+ # type: (Text, Text, Text) -> Text
+ split = urllib.parse.urlsplit(docid)
sp = split.fragment.split(u"/")
n = self.scoped_ref_fields[field]
while n > 0 and len(sp) > 0:
@@ -842,7 +880,7 @@ class Loader(object):
tried = []
while True:
sp.append(link)
- url = urlparse.urlunsplit((
+ url = urllib.parse.urlunsplit((
split.scheme, split.netloc, split.path, split.query,
u"/".join(sp)))
tried.append(url)
@@ -855,13 +893,14 @@ class Loader(object):
raise validate.ValidationException(
"Field `%s` references unknown identifier `%s`, tried %s" % (field, link, ", ".join(tried)))
- def validate_link(self, field, link, docid):
- # type: (unicode, FieldType, unicode) -> FieldType
+ def validate_link(self, field, link, docid, all_doc_ids):
+ # type: (Text, FieldType, Text, Dict[Text, Text]) -> FieldType
if field in self.nolinkcheck:
return link
- if isinstance(link, (str, unicode)):
+ if isinstance(link, (str, six.text_type)):
if field in self.vocab_fields:
- if link not in self.vocab and link not in self.idx and link not in self.rvocab:
+ if (link not in self.vocab and link not in self.idx
+ and link not in self.rvocab):
if field in self.scoped_ref_fields:
return self.validate_scoped(field, link, docid)
elif not self.check_exists(link):
@@ -872,34 +911,37 @@ class Loader(object):
return self.validate_scoped(field, link, docid)
elif not self.check_exists(link):
raise validate.ValidationException(
- "Field `%s` contains undefined reference to `%s`" % (field, link))
+ "Field `%s` contains undefined reference to `%s`"
+ % (field, link))
elif isinstance(link, CommentedSeq):
errors = []
for n, i in enumerate(link):
try:
- link[n] = self.validate_link(field, i, docid)
+ link[n] = self.validate_link(field, i, docid, all_doc_ids)
except validate.ValidationException as v:
errors.append(v)
- if errors:
+ if bool(errors):
raise validate.ValidationException(
- "\n".join([unicode(e) for e in errors]))
+ "\n".join([six.text_type(e) for e in errors]))
elif isinstance(link, CommentedMap):
- self.validate_links(link, docid)
+ self.validate_links(link, docid, all_doc_ids)
else:
raise validate.ValidationException(
- "`%s` field is %s, expected string, list, or a dict." % (field, type(link).__name__))
+ "`%s` field is %s, expected string, list, or a dict."
+ % (field, type(link).__name__))
return link
- def getid(self, d): # type: (Any) -> unicode
+ def getid(self, d): # type: (Any) -> Optional[Text]
if isinstance(d, dict):
for i in self.identifiers:
if i in d:
- if isinstance(d[i], (str, unicode)):
- return d[i]
+ idd = d[i]
+ if isinstance(idd, (str, six.text_type)):
+ return idd
return None
- def validate_links(self, document, base_url):
- # type: (Union[CommentedMap, CommentedSeq, unicode], unicode) -> None
+ def validate_links(self, document, base_url, all_doc_ids):
+ # type: (Union[CommentedMap, CommentedSeq, Text, None], Text, Dict[Text, Text]) -> None
docid = self.getid(document)
if not docid:
docid = base_url
@@ -913,38 +955,48 @@ class Loader(object):
for d in self.url_fields:
sl = SourceLine(document, d, validate.ValidationException)
if d in document and d not in self.identity_links:
- document[d] = self.validate_link(d, document[d], docid)
+ document[d] = self.validate_link(d, document[d], docid, all_doc_ids)
+ for identifier in self.identifiers: # validate that each id is defined uniquely
+ if identifier in document:
+ sl = SourceLine(document, identifier, validate.ValidationException)
+ if document[identifier] in all_doc_ids and sl.makeLead() != all_doc_ids[document[identifier]]:
+ raise validate.ValidationException(
+ "%s object %s `%s` previously defined" % (all_doc_ids[document[identifier]], identifier, relname(document[identifier]), ))
+ else:
+ all_doc_ids[document[identifier]] = sl.makeLead()
+ break
except validate.ValidationException as v:
- errors.append(sl.makeError(unicode(v)))
+ errors.append(sl.makeError(six.text_type(v)))
if hasattr(document, "iteritems"):
- iterator = document.iteritems()
+ iterator = six.iteritems(document)
else:
- iterator = document.items()
+ iterator = list(document.items())
else:
return
for key, val in iterator:
sl = SourceLine(document, key, validate.ValidationException)
try:
- self.validate_links(val, docid)
+ self.validate_links(val, docid, all_doc_ids)
except validate.ValidationException as v:
if key not in self.nolinkcheck:
docid2 = self.getid(val)
- if docid2:
- errors.append(sl.makeError("checking object `%s`\n%s" % (
- relname(docid2), validate.indent(unicode(v)))))
+ if docid2 is not None:
+ errors.append(sl.makeError("checking object `%s`\n%s"
+ % (relname(docid2), validate.indent(six.text_type(v)))))
else:
- if isinstance(key, basestring):
+ if isinstance(key, six.string_types):
errors.append(sl.makeError("checking field `%s`\n%s" % (
- key, validate.indent(unicode(v)))))
+ key, validate.indent(six.text_type(v)))))
else:
errors.append(sl.makeError("checking item\n%s" % (
- validate.indent(unicode(v)))))
-
- if errors:
+ validate.indent(six.text_type(v)))))
+ else:
+ _logger.warn( validate.indent(six.text_type(v)))
+ if bool(errors):
if len(errors) > 1:
raise validate.ValidationException(
- u"\n".join([unicode(e) for e in errors]))
+ u"\n".join([six.text_type(e) for e in errors]))
else:
raise errors[0]
return
@@ -956,7 +1008,7 @@ def _copy_dict_without_key(from_dict, filtered_key):
# type: (D, Any) -> D
new_dict = copy.copy(from_dict)
if filtered_key in new_dict:
- del new_dict[filtered_key] # type: ignore
+ del new_dict[filtered_key]
if isinstance(from_dict, CommentedMap):
new_dict.lc.data = copy.copy(from_dict.lc.data)
new_dict.lc.filename = from_dict.lc.filename
diff --git a/schema_salad/schema.py b/schema_salad/schema.py
index 342ec46..f1ca9af 100644
--- a/schema_salad/schema.py
+++ b/schema_salad/schema.py
@@ -1,6 +1,7 @@
+from __future__ import absolute_import
import avro
import copy
-from .add_dictlist import add_dictlist
+from schema_salad.utils import add_dictlist, aslist, flatten
import sys
import pprint
from pkg_resources import resource_stream
@@ -8,19 +9,23 @@ import ruamel.yaml as yaml
import avro.schema
from . import validate
import json
-import urlparse
import os
-AvroSchemaFromJSONData = avro.schema.make_avsc_object
-# AvroSchemaFromJSONData=avro.schema.SchemaFromJSONData
+
+import six
+from six.moves import urllib
+
+if six.PY3:
+ AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
+else:
+ AvroSchemaFromJSONData = avro.schema.make_avsc_object
+
from avro.schema import Names, SchemaParseException
from . import ref_resolver
from .ref_resolver import Loader, DocumentType
-from .flatten import flatten
import logging
-from .aslist import aslist
from . import jsonld_context
from .sourceline import SourceLine, strip_dup_lineno, add_lc_filename, bullets, relname
-from typing import Any, AnyStr, cast, Dict, List, Tuple, TypeVar, Union
+from typing import cast, Any, AnyStr, Dict, List, Set, Tuple, TypeVar, Union, Text
from ruamel.yaml.comments import CommentedSeq, CommentedMap
_logger = logging.getLogger("salad")
@@ -45,11 +50,19 @@ salad_files = ('metaschema.yml',
'link_res_proc.yml',
'vocab_res_schema.yml',
'vocab_res_src.yml',
- 'vocab_res_proc.yml')
+ 'vocab_res_proc.yml',
+ 'map_res.yml',
+ 'map_res_schema.yml',
+ 'map_res_src.yml',
+ 'map_res_proc.yml',
+ 'typedsl_res.yml',
+ 'typedsl_res_schema.yml',
+ 'typedsl_res_src.yml',
+ 'typedsl_res_proc.yml')
def get_metaschema():
- # type: () -> Tuple[Names, List[Dict[unicode, Any]], Loader]
+ # type: () -> Tuple[Names, List[Dict[Text, Any]], Loader]
loader = ref_resolver.Loader({
"Any": "https://w3id.org/cwl/salad#Any",
"ArraySchema": "https://w3id.org/cwl/salad#ArraySchema",
@@ -163,7 +176,7 @@ def get_metaschema():
loader.cache["https://w3id.org/cwl/salad"] = rs.read()
rs.close()
- j = yaml.round_trip_load(loader.cache["https://w3id.org/cwl/salad"]) # type: ignore
+ j = yaml.round_trip_load(loader.cache["https://w3id.org/cwl/salad"])
add_lc_filename(j, "metaschema.yml")
j, _ = loader.resolve_all(j, "https://w3id.org/cwl/salad#")
@@ -178,10 +191,10 @@ def get_metaschema():
return (sch_names, j, loader)
-def load_schema(schema_ref, # type: Union[CommentedMap, CommentedSeq, unicode]
+def load_schema(schema_ref, # type: Union[CommentedMap, CommentedSeq, Text]
cache=None # type: Dict
):
- # type: (...) -> Tuple[Loader, Union[Names, SchemaParseException], Dict[unicode, Any], Loader]
+ # type: (...) -> Tuple[Loader, Union[Names, SchemaParseException], Dict[Text, Any], Loader]
"""Load a schema that can be used to validate documents using load_and_validate.
return document_loader, avsc_names, schema_metadata, metaschema_loader"""
@@ -212,10 +225,10 @@ def load_schema(schema_ref, # type: Union[CommentedMap, CommentedSeq, unicode]
def load_and_validate(document_loader, # type: Loader
avsc_names, # type: Names
- document, # type: Union[CommentedMap, unicode]
+ document, # type: Union[CommentedMap, Text]
strict # type: bool
):
- # type: (...) -> Tuple[Any, Dict[unicode, Any]]
+ # type: (...) -> Tuple[Any, Dict[Text, Any]]
"""Load a document and validate it with the provided schema.
return data, metadata
@@ -234,23 +247,23 @@ def load_and_validate(document_loader, # type: Loader
validationErrors = u""
try:
- document_loader.validate_links(data, u"")
+ document_loader.validate_links(data, u"", {})
except validate.ValidationException as v:
- validationErrors = unicode(v) + "\n"
+ validationErrors = six.text_type(v) + "\n"
try:
validate_doc(avsc_names, data, document_loader, strict, source=source)
except validate.ValidationException as v:
- validationErrors += unicode(v)
+ validationErrors += six.text_type(v)
- if validationErrors:
+ if validationErrors != u"":
raise validate.ValidationException(validationErrors)
return data, metadata
def validate_doc(schema_names, # type: Names
- doc, # type: Union[Dict[unicode, Any], List[Dict[unicode, Any]], unicode]
+ doc, # type: Union[Dict[Text, Any], List[Dict[Text, Any]], Text, None]
loader, # type: Loader
strict, # type: bool
source=None
@@ -284,7 +297,7 @@ def validate_doc(schema_names, # type: Names
anyerrors = []
for pos, item in enumerate(validate_doc):
- sl = SourceLine(validate_doc, pos, unicode)
+ sl = SourceLine(validate_doc, pos, six.text_type)
success = False
for r in roots:
success = validate.validate_ex(
@@ -294,7 +307,7 @@ def validate_doc(schema_names, # type: Names
break
if not success:
- errors = [] # type: List[unicode]
+ errors = [] # type: List[Text]
for r in roots:
if hasattr(r, "get_prop"):
name = r.get_prop(u"name")
@@ -323,16 +336,15 @@ def validate_doc(schema_names, # type: Names
break
anyerrors.append(u"%s\n%s" %
(objerr, validate.indent(bullets(errors, "- "))))
- if anyerrors:
+ if len(anyerrors) > 0:
raise validate.ValidationException(
strip_dup_lineno(bullets(anyerrors, "* ")))
def replace_type(items, spec, loader, found):
- # type: (Any, Dict[unicode, Any], Loader, Set[unicode]) -> Any
+ # type: (Any, Dict[Text, Any], Loader, Set[Text]) -> Any
""" Go through and replace types in the 'spec' mapping"""
- items = copy.deepcopy(items)
if isinstance(items, dict):
# recursively check these fields for types to replace
if "type" in items and items["type"] in ("record", "enum"):
@@ -342,6 +354,7 @@ def replace_type(items, spec, loader, found):
else:
found.add(items["name"])
+ items = copy.copy(items)
for n in ("type", "items", "fields"):
if n in items:
items[n] = replace_type(items[n], spec, loader, found)
@@ -352,7 +365,7 @@ def replace_type(items, spec, loader, found):
elif isinstance(items, list):
# recursively transform list
return [replace_type(i, spec, loader, found) for i in items]
- elif isinstance(items, (str, unicode)):
+ elif isinstance(items, (str, six.text_type)):
# found a string which is a symbol corresponding to a type.
replace_with = None
if items in loader.vocab:
@@ -370,8 +383,8 @@ def replace_type(items, spec, loader, found):
def avro_name(url): # type: (AnyStr) -> AnyStr
- doc_url, frg = urlparse.urldefrag(url)
- if frg:
+ doc_url, frg = urllib.parse.urldefrag(url)
+ if frg != '':
if '/' in frg:
return frg[frg.rindex('/') + 1:]
else:
@@ -379,19 +392,20 @@ def avro_name(url): # type: (AnyStr) -> AnyStr
return url
-Avro = TypeVar('Avro', Dict[unicode, Any], List[Any], unicode)
+Avro = TypeVar('Avro', Dict[Text, Any], List[Any], Text)
def make_valid_avro(items, # type: Avro
- alltypes, # type: Dict[unicode, Dict[unicode, Any]]
- found, # type: Set[unicode]
+ alltypes, # type: Dict[Text, Dict[Text, Any]]
+ found, # type: Set[Text]
union=False # type: bool
):
- # type: (...) -> Union[Avro, Dict]
- items = copy.deepcopy(items)
+ # type: (...) -> Union[Avro, Dict, Text]
if isinstance(items, dict):
+ items = copy.copy(items)
if items.get("name"):
- items["name"] = avro_name(items["name"])
+ if items.get("inVocab", True):
+ items["name"] = avro_name(items["name"])
if "type" in items and items["type"] in ("https://w3id.org/cwl/salad#record", "https://w3id.org/cwl/salad#enum", "record", "enum"):
if (hasattr(items, "get") and items.get("abstract")) or ("abstract"
@@ -402,7 +416,7 @@ def make_valid_avro(items, # type: Avro
"Named schemas must have a non-empty name: %s" % items)
if items["name"] in found:
- return items["name"]
+ return cast(Text, items["name"])
else:
found.add(items["name"])
for n in ("type", "items", "values", "fields"):
@@ -415,45 +429,57 @@ def make_valid_avro(items, # type: Avro
if isinstance(items, list):
ret = []
for i in items:
- ret.append(make_valid_avro(i, alltypes, found, union=union))
+ ret.append(make_valid_avro(i, alltypes, found, union=union)) # type: ignore
return ret
- if union and isinstance(items, (str, unicode)):
+ if union and isinstance(items, six.string_types):
if items in alltypes and avro_name(items) not in found:
return cast(Dict, make_valid_avro(alltypes[items], alltypes, found,
union=union))
items = avro_name(items)
return items
+def deepcopy_strip(item): # type: (Any) -> Any
+ """Make a deep copy of list and dict objects.
+
+ Intentionally do not copy attributes. This is to discard CommentedMap and
+ CommentedSeq metadata which is very expensive with regular copy.deepcopy.
+
+ """
+
+ if isinstance(item, dict):
+ return {k: deepcopy_strip(v) for k,v in six.iteritems(item)}
+ elif isinstance(item, list):
+ return [deepcopy_strip(k) for k in item]
+ else:
+ return item
def extend_and_specialize(items, loader):
- # type: (List[Dict[unicode, Any]], Loader) -> List[Dict[unicode, Any]]
+ # type: (List[Dict[Text, Any]], Loader) -> List[Dict[Text, Any]]
"""Apply 'extend' and 'specialize' to fully materialize derived record
types."""
- types = {} # type: Dict[unicode, Any]
- for t in items:
- types[t["name"]] = t
+ items = deepcopy_strip(items)
+ types = {t["name"]: t for t in items} # type: Dict[Text, Any]
n = []
for t in items:
- t = copy.deepcopy(t)
if "extends" in t:
- spec = {} # type: Dict[unicode, unicode]
+ spec = {} # type: Dict[Text, Text]
if "specialize" in t:
for sp in aslist(t["specialize"]):
spec[sp["specializeFrom"]] = sp["specializeTo"]
- exfields = [] # type: List[unicode]
- exsym = [] # type: List[unicode]
+ exfields = [] # type: List[Text]
+ exsym = [] # type: List[Text]
for ex in aslist(t["extends"]):
if ex not in types:
raise Exception("Extends %s in %s refers to invalid base type" % (
t["extends"], t["name"]))
- basetype = copy.deepcopy(types[ex])
+ basetype = copy.copy(types[ex])
if t["type"] == "record":
- if spec:
+ if len(spec) > 0:
basetype["fields"] = replace_type(
basetype.get("fields", []), spec, loader, set())
@@ -466,10 +492,11 @@ def extend_and_specialize(items, loader):
exsym.extend(basetype.get("symbols", []))
if t["type"] == "record":
+ t = copy.copy(t)
exfields.extend(t.get("fields", []))
t["fields"] = exfields
- fieldnames = set() # type: Set[unicode]
+ fieldnames = set() # type: Set[Text]
for field in t["fields"]:
if field["name"] in fieldnames:
raise validate.ValidationException(
@@ -477,6 +504,7 @@ def extend_and_specialize(items, loader):
else:
fieldnames.add(field["name"])
elif t["type"] == "enum":
+ t = copy.copy(t)
exsym.extend(t.get("symbols", []))
t["symbol"] = exsym
@@ -488,7 +516,7 @@ def extend_and_specialize(items, loader):
for t in n:
ex_types[t["name"]] = t
- extended_by = {} # type: Dict[unicode, unicode]
+ extended_by = {} # type: Dict[Text, Text]
for t in n:
if "extends" in t:
for ex in aslist(t["extends"]):
@@ -507,16 +535,15 @@ def extend_and_specialize(items, loader):
return n
-
-def make_avro_schema(i, # type: List[Dict[unicode, Any]]
+def make_avro_schema(i, # type: List[Dict[Text, Any]]
loader # type: Loader
):
- # type: (...) -> Tuple[Union[Names, SchemaParseException], List[Dict[unicode, Any]]]
+ # type: (...) -> Tuple[Union[Names, SchemaParseException], List[Dict[Text, Any]]]
names = avro.schema.Names()
j = extend_and_specialize(i, loader)
- name_dict = {} # type: Dict[unicode, Dict[unicode, Any]]
+ name_dict = {} # type: Dict[Text, Dict[Text, Any]]
for t in j:
name_dict[t["name"]] = t
j2 = make_valid_avro(j, name_dict, set())
diff --git a/schema_salad/sourceline.py b/schema_salad/sourceline.py
index e09171c..21e57c1 100644
--- a/schema_salad/sourceline.py
+++ b/schema_salad/sourceline.py
@@ -1,3 +1,4 @@
+from __future__ import absolute_import
import ruamel.yaml
from ruamel.yaml.comments import CommentedBase, CommentedMap, CommentedSeq
import re
@@ -5,6 +6,7 @@ import os
from typing import (Any, AnyStr, Callable, cast, Dict, List, Iterable, Tuple,
TypeVar, Union, Text)
+import six
lineno_re = re.compile(u"^(.*?:[0-9]+:[0-9]+: )(( *)(.*))")
@@ -15,19 +17,19 @@ def _add_lc_filename(r, source): # type: (ruamel.yaml.comments.CommentedBase, A
for d in r:
_add_lc_filename(d, source)
elif isinstance(r, dict):
- for d in r.itervalues():
+ for d in six.itervalues(r):
_add_lc_filename(d, source)
-def relname(source): # type: (AnyStr) -> AnyStr
+def relname(source): # type: (Text) -> Text
if source.startswith("file://"):
source = source[7:]
source = os.path.relpath(source)
return source
-def add_lc_filename(r, source): # type: (ruamel.yaml.comments.CommentedBase, AnyStr) -> None
+def add_lc_filename(r, source): # type: (ruamel.yaml.comments.CommentedBase, Text) -> None
_add_lc_filename(r, relname(source))
-def reflow(text, maxline, shift=""): # type: (AnyStr, int, AnyStr) -> AnyStr
+def reflow(text, maxline, shift=""): # type: (Text, int, Text) -> Text
if maxline < 20:
maxline = 20
if len(text) > maxline:
@@ -46,7 +48,7 @@ def indent(v, nolead=False, shift=u" ", bullet=u" "): # type: (Text, bool, Te
else:
def lineno(i, l): # type: (int, Text) -> Text
r = lineno_re.match(l)
- if r:
+ if bool(r):
return r.group(1) + (bullet if i == 0 else shift) + r.group(2)
else:
return (bullet if i == 0 else shift) + l
@@ -79,7 +81,7 @@ def strip_dup_lineno(text, maxline=None): # type: (Text, int) -> Text
msg.append(" " * len(g.group(1)) + g2)
return "\n".join(msg)
-def cmap(d, lc=None, fn=None): # type: (Union[int, float, str, unicode, Dict, List], List[int], unicode) -> Union[int, float, str, unicode, CommentedMap, CommentedSeq]
+def cmap(d, lc=None, fn=None): # type: (Union[int, float, str, Text, Dict, List], List[int], Text) -> Union[int, float, str, Text, CommentedMap, CommentedSeq]
if lc is None:
lc = [0, 0, 0, 0]
if fn is None:
@@ -87,7 +89,7 @@ def cmap(d, lc=None, fn=None): # type: (Union[int, float, str, unicode, Dict, L
if isinstance(d, CommentedMap):
fn = d.lc.filename if hasattr(d.lc, "filename") else fn
- for k,v in d.iteritems():
+ for k,v in six.iteritems(d):
if k in d.lc.data:
d[k] = cmap(v, lc=d.lc.data[k], fn=fn)
else:
@@ -132,33 +134,40 @@ def cmap(d, lc=None, fn=None): # type: (Union[int, float, str, unicode, Dict, L
return d
class SourceLine(object):
- def __init__(self, item, key=None, raise_type=unicode): # type: (Any, Any, Callable) -> None
+ def __init__(self, item, key=None, raise_type=six.text_type): # type: (Any, Any, Callable) -> None
self.item = item
self.key = key
self.raise_type = raise_type
- def __enter__(self):
+ def __enter__(self): # type: () -> SourceLine
return self
- def __exit__(self, exc_type, exc_value, traceback):
+ def __exit__(self,
+ exc_type, # type: Any
+ exc_value, # type: Any
+ traceback # type: Any
+ ): # -> Any
if not exc_value:
return
- raise self.makeError(unicode(exc_value))
+ raise self.makeError(six.text_type(exc_value))
+
+ def makeLead(self): # type: () -> Text
+ if self.key is None or self.item.lc.data is None or self.key not in self.item.lc.data:
+ return "%s:%i:%i:" % (self.item.lc.filename if hasattr(self.item.lc, "filename") else "",
+ (self.item.lc.line or 0)+1,
+ (self.item.lc.col or 0)+1)
+ else:
+ return "%s:%i:%i:" % (self.item.lc.filename if hasattr(self.item.lc, "filename") else "",
+ (self.item.lc.data[self.key][0] or 0)+1,
+ (self.item.lc.data[self.key][1] or 0)+1)
def makeError(self, msg): # type: (Text) -> Any
if not isinstance(self.item, ruamel.yaml.comments.CommentedBase):
return self.raise_type(msg)
errs = []
- if self.key is None or self.item.lc.data is None or self.key not in self.item.lc.data:
- lead = "%s:%i:%i:" % (self.item.lc.filename,
- self.item.lc.line+1,
- self.item.lc.col+1)
- else:
- lead = "%s:%i:%i:" % (self.item.lc.filename,
- self.item.lc.data[self.key][0]+1,
- self.item.lc.data[self.key][1]+1)
+ lead = self.makeLead()
for m in msg.splitlines():
- if lineno_re.match(m):
+ if bool(lineno_re.match(m)):
errs.append(m)
else:
errs.append("%s %s" % (lead, m))
diff --git a/schema_salad/tests/.coverage b/schema_salad/tests/.coverage
deleted file mode 100644
index b4ab5e5..0000000
--- a/schema_salad/tests/.coverage
+++ /dev/null
@@ -1 +0,0 @@
-!coverage.py: This is a private format, don't read it directly!{"lines": {"/home/peter/work/salad/schema_salad/validate.py": [1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 13, 15, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 31, 32, 33, 37, 38, 39, 41, 43, 44, 48, 51, 52, 54, 56, 57, 58, 60, 63, 64, 65, 66, 72, 73, 74, 75, 79, 80, 82, 83, 91, 92, 93, 94, 100, 109, 118, 119, 127, 128, 129, 131, 132, 133, 135, 136, 137, 138, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 155, 157, 158, 160, [...]
\ No newline at end of file
diff --git a/schema_salad/tests/test_cli_args.py b/schema_salad/tests/test_cli_args.py
new file mode 100644
index 0000000..16b7e5c
--- /dev/null
+++ b/schema_salad/tests/test_cli_args.py
@@ -0,0 +1,41 @@
+from __future__ import absolute_import
+import unittest
+import sys
+
+import schema_salad.main as cli_parser
+
+# for capturing print() output
+from contextlib import contextmanager
+from six import StringIO
+
+ at contextmanager
+def captured_output():
+ new_out, new_err = StringIO(), StringIO()
+ old_out, old_err = sys.stdout, sys.stderr
+ try:
+ sys.stdout, sys.stderr = new_out, new_err
+ yield sys.stdout, sys.stderr
+ finally:
+ sys.stdout, sys.stderr = old_out, old_err
+
+
+""" test different sets of command line arguments"""
+class ParseCliArgs(unittest.TestCase):
+
+ def test_version(self):
+ args = [["--version"], ["-v"]]
+ for arg in args:
+ with captured_output() as (out, err):
+ cli_parser.main(arg)
+
+ response = out.getvalue().strip() # capture output and strip newline
+ self.assertTrue("Current version" in response)
+
+ def test_empty_input(self):
+ # running schema_salad tool wihtout any args
+ args = []
+ with captured_output() as (out, err):
+ cli_parser.main(args)
+
+ response = out.getvalue().strip()
+ self.assertTrue("error: too few arguments" in response)
diff --git a/schema_salad/tests/test_errors.py b/schema_salad/tests/test_errors.py
index 25a5eea..590984a 100644
--- a/schema_salad/tests/test_errors.py
+++ b/schema_salad/tests/test_errors.py
@@ -1,15 +1,16 @@
+from __future__ import absolute_import
+from __future__ import print_function
from .util import get_data
import unittest
-from typing import cast
from schema_salad.schema import load_schema, load_and_validate
from schema_salad.validate import ValidationException
from avro.schema import Names
+import six
class TestErrors(unittest.TestCase):
def test_errors(self):
document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(
get_data(u"tests/test_schema/CommonWorkflowLanguage.yml"))
- avsc_names = cast(Names, avsc_names)
for t in ("test_schema/test1.cwl",
"test_schema/test2.cwl",
@@ -21,11 +22,14 @@ class TestErrors(unittest.TestCase):
"test_schema/test8.cwl",
"test_schema/test9.cwl",
"test_schema/test10.cwl",
- "test_schema/test11.cwl"):
+ "test_schema/test11.cwl",
+ "test_schema/test12.cwl",
+ "test_schema/test13.cwl",
+ "test_schema/test14.cwl"):
with self.assertRaises(ValidationException):
try:
load_and_validate(document_loader, avsc_names,
- unicode(get_data("tests/"+t)), True)
+ six.text_type(get_data("tests/"+t)), True)
except ValidationException as e:
- print "\n", e
+ print("\n", e)
raise
diff --git a/schema_salad/tests/test_errors.py~ b/schema_salad/tests/test_errors.py~
deleted file mode 100644
index 01058d8..0000000
--- a/schema_salad/tests/test_errors.py~
+++ /dev/null
@@ -1 +0,0 @@
-g
diff --git a/schema_salad/tests/test_examples.py b/schema_salad/tests/test_examples.py
index 6b0277c..4ffdd0d 100644
--- a/schema_salad/tests/test_examples.py
+++ b/schema_salad/tests/test_examples.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+from __future__ import print_function
from .util import get_data
import unittest
import schema_salad.ref_resolver
@@ -47,7 +49,7 @@ class TestSchemas(unittest.TestCase):
# "edam:has_format": "edam:format_1915"
# }, "")
- # self.assertEquals(ra, {
+ # self.assertEqual(ra, {
# "$schemas": ["tests/EDAM.owl"],
# "$namespaces": {"edam": "http://edamontology.org/"},
# 'http://edamontology.org/has_format': 'http://edamontology.org/format_1915'
@@ -112,12 +114,12 @@ class TestSchemas(unittest.TestCase):
self.assertEqual("http://example2.com/#stuff", ra["id"])
for item in ra["inputs"]:
if item["a"] == 2:
- self.assertEquals(
+ self.assertEqual(
'http://example2.com/#stuff/zing', item["id"])
else:
- self.assertEquals('http://example2.com/#stuff/zip', item["id"])
- self.assertEquals(['http://example2.com/#stuff/out'], ra['outputs'])
- self.assertEquals({'n': 9}, ra['other'])
+ self.assertEqual('http://example2.com/#stuff/zip', item["id"])
+ self.assertEqual(['http://example2.com/#stuff/out'], ra['outputs'])
+ self.assertEqual({'n': 9}, ra['other'])
def test_scoped_ref(self):
ldr = schema_salad.ref_resolver.Loader({})
@@ -181,7 +183,7 @@ class TestSchemas(unittest.TestCase):
}
}), "http://example2.com/")
- self.assertEquals(
+ self.assertEqual(
{'inputs': [{
'id': 'http://example2.com/#inp',
'type': 'string'
@@ -360,15 +362,16 @@ class TestSchemas(unittest.TestCase):
def test_fragment(self):
ldr = schema_salad.ref_resolver.Loader({"id": "@id"})
b, _ = ldr.resolve_ref(get_data("tests/frag.yml#foo2"))
- self.assertEquals({"id": b["id"], "bar":"b2"}, b)
+ self.assertEqual({"id": b["id"], "bar":"b2"}, b)
def test_file_uri(self):
# Note: this test probably won't pass on Windows. Someone with a
# windows box should add an alternate test.
- self.assertEquals("file:///foo/bar%20baz/quux", schema_salad.ref_resolver.file_uri("/foo/bar baz/quux"))
- self.assertEquals("/foo/bar baz/quux", schema_salad.ref_resolver.uri_file_path("file:///foo/bar%20baz/quux"))
- self.assertEquals("file:///foo/bar%20baz/quux#zing%20zong", schema_salad.ref_resolver.file_uri("/foo/bar baz/quux#zing zong"))
- self.assertEquals("/foo/bar baz/quux#zing zong", schema_salad.ref_resolver.uri_file_path("file:///foo/bar%20baz/quux#zing%20zong"))
+ self.assertEqual("file:///foo/bar%20baz/quux", schema_salad.ref_resolver.file_uri("/foo/bar baz/quux"))
+ self.assertEqual("/foo/bar baz/quux", schema_salad.ref_resolver.uri_file_path("file:///foo/bar%20baz/quux"))
+ self.assertEqual("file:///foo/bar%20baz/quux%23zing%20zong", schema_salad.ref_resolver.file_uri("/foo/bar baz/quux#zing zong"))
+ self.assertEqual("file:///foo/bar%20baz/quux#zing%20zong", schema_salad.ref_resolver.file_uri("/foo/bar baz/quux#zing zong", split_frag=True))
+ self.assertEqual("/foo/bar baz/quux#zing zong", schema_salad.ref_resolver.uri_file_path("file:///foo/bar%20baz/quux#zing%20zong"))
if __name__ == '__main__':
diff --git a/schema_salad/tests/test_fetch.py b/schema_salad/tests/test_fetch.py
index 8fb9e5a..6f36951 100644
--- a/schema_salad/tests/test_fetch.py
+++ b/schema_salad/tests/test_fetch.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+from __future__ import print_function
import unittest
import schema_salad.ref_resolver
import schema_salad.main
@@ -7,7 +9,8 @@ import rdflib
import ruamel.yaml as yaml
import json
import os
-import urlparse
+from typing import Text
+from six.moves import urllib
class TestFetcher(unittest.TestCase):
def test_fetcher(self):
@@ -15,7 +18,7 @@ class TestFetcher(unittest.TestCase):
def __init__(self, a, b):
pass
- def fetch_text(self, url): # type: (unicode) -> unicode
+ def fetch_text(self, url): # type: (Text) -> Text
if url == "keep:abc+123/foo.txt":
return "hello: keepfoo"
if url.endswith("foo.txt"):
@@ -23,21 +26,21 @@ class TestFetcher(unittest.TestCase):
else:
raise RuntimeError("Not foo.txt")
- def check_exists(self, url): # type: (unicode) -> bool
+ def check_exists(self, url): # type: (Text) -> bool
if url.endswith("foo.txt"):
return True
else:
return False
def urljoin(self, base, url):
- urlsp = urlparse.urlsplit(url)
+ urlsp = urllib.parse.urlsplit(url)
if urlsp.scheme:
return url
- basesp = urlparse.urlsplit(base)
+ basesp = urllib.parse.urlsplit(base)
if basesp.scheme == "keep":
return base + "/" + url
- return urlparse.urljoin(base, url)
+ return urllib.parse.urljoin(base, url)
loader = schema_salad.ref_resolver.Loader({}, fetcher_constructor=TestFetcher)
self.assertEqual({"hello": "foo"}, loader.resolve_ref("foo.txt")[0])
@@ -52,6 +55,6 @@ class TestFetcher(unittest.TestCase):
loader = schema_salad.ref_resolver.Loader({})
foo = "file://%s/foo.txt" % os.getcwd()
loader.cache.update({foo: "hello: foo"})
- print loader.cache
+ print(loader.cache)
self.assertEqual({"hello": "foo"}, loader.resolve_ref("foo.txt")[0])
self.assertTrue(loader.check_exists(foo))
diff --git a/schema_salad/tests/test_fetch.py~ b/schema_salad/tests/test_fetch.py~
deleted file mode 100644
index 422d945..0000000
--- a/schema_salad/tests/test_fetch.py~
+++ /dev/null
@@ -1,13 +0,0 @@
-import unittest
-import schema_salad.ref_resolver
-import schema_salad.main
-import schema_salad.schema
-from schema_salad.jsonld_context import makerdf
-from pkg_resources import Requirement, resource_filename, ResolutionError # type: ignore
-import rdflib
-import ruamel.yaml as yaml
-import json
-import os
-
-class TestFetcher(unittest.TestCase):
- def test_schemas(self):
diff --git a/schema_salad/tests/test_ref_resolver.py b/schema_salad/tests/test_ref_resolver.py
new file mode 100644
index 0000000..fdb5e61
--- /dev/null
+++ b/schema_salad/tests/test_ref_resolver.py
@@ -0,0 +1,55 @@
+"""Test the ref_resolver module."""
+
+from __future__ import absolute_import
+import shutil
+import tempfile
+
+import pytest # type: ignore
+
+ at pytest.fixture
+def tmp_dir_fixture(request):
+ d = tempfile.mkdtemp()
+
+ @request.addfinalizer
+ def teardown():
+ shutil.rmtree(d)
+ return d
+
+def test_Loader_initialisation_for_HOME_env_var(tmp_dir_fixture):
+ import os
+ from schema_salad.ref_resolver import Loader
+ from requests import Session
+
+ # Ensure HOME is set.
+ os.environ["HOME"] = tmp_dir_fixture
+
+ loader = Loader(ctx={})
+ assert isinstance(loader.session, Session)
+
+def test_Loader_initialisation_for_TMP_env_var(tmp_dir_fixture):
+ import os
+ from schema_salad.ref_resolver import Loader
+ from requests import Session
+
+ # Ensure HOME is missing.
+ if "HOME" in os.environ:
+ del os.environ["HOME"]
+ # Ensure TMP is present.
+ os.environ["TMP"] = tmp_dir_fixture
+
+ loader = Loader(ctx={})
+ assert isinstance(loader.session, Session)
+
+def test_Loader_initialisation_with_neither_TMP_HOME_set(tmp_dir_fixture):
+ import os
+ from schema_salad.ref_resolver import Loader
+ from requests import Session
+
+ # Ensure HOME is missing.
+ if "HOME" in os.environ:
+ del os.environ["HOME"]
+ if "TMP" in os.environ:
+ del os.environ["TMP"]
+
+ loader = Loader(ctx={})
+ assert isinstance(loader.session, Session)
diff --git a/schema_salad/tests/test_schema/test12.cwl b/schema_salad/tests/test_schema/test12.cwl
new file mode 100644
index 0000000..d994e7c
--- /dev/null
+++ b/schema_salad/tests/test_schema/test12.cwl
@@ -0,0 +1,16 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+inputs:
+ - id: example_flag
+ type: boolean
+ inputBinding:
+ position: 1
+ prefix: -f
+ - id: example_flag
+ type: int
+ inputBinding:
+ position: 3
+ prefix: --example-string
+
+outputs: []
diff --git a/schema_salad/tests/test_schema/test13.cwl b/schema_salad/tests/test_schema/test13.cwl
new file mode 100644
index 0000000..caa274d
--- /dev/null
+++ b/schema_salad/tests/test_schema/test13.cwl
@@ -0,0 +1,20 @@
+cwlVersion: v1.0
+class: Workflow
+inputs:
+ example_flag:
+ type: boolean
+ inputBinding:
+ position: 1
+ prefix: -f
+
+outputs: []
+
+steps:
+ example_flag:
+ in: []
+ out: []
+ run:
+ id: blah
+ class: CommandLineTool
+ inputs: []
+ outputs: []
\ No newline at end of file
diff --git a/schema_salad/tests/test_schema/test14.cwl b/schema_salad/tests/test_schema/test14.cwl
new file mode 100644
index 0000000..729ee83
--- /dev/null
+++ b/schema_salad/tests/test_schema/test14.cwl
@@ -0,0 +1,11 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+inputs:
+ example_flag:
+ type: boolean
+ inputBinding:
+ position: 1
+ prefix: -f
+outputs:
+ example_flag: int
diff --git a/schema_salad/tests/test_validate.pyx b/schema_salad/tests/test_validate.pyx
deleted file mode 100644
index b37127a..0000000
--- a/schema_salad/tests/test_validate.pyx
+++ /dev/null
@@ -1,71 +0,0 @@
-import unittest
-import json
-from schema_salad.schema import load_schema
-from schema_salad.validate import validate_ex
-from schema_salad.sourceline import cmap
-
-class TestValidate(unittest.TestCase):
- schema = cmap({"name": "_", "$graph":[{
- "name": "File",
- "type": "record",
- "fields": [{
- "name": "class",
- "type": {
- "type": "enum",
- "name": "File_class",
- "symbols": ["#_/File"]
- },
- "jsonldPredicate": {
- "_id": "@type",
- "_type": "@vocab"
- }
- }, {
- "name": "location",
- "type": "string",
- "jsonldPredicate": "_:location"
- }]
- }, {
- "name": "Directory",
- "type": "record",
- "fields": [{
- "name": "class",
- "type": {
- "type": "enum",
- "name": "Directory_class",
- "symbols": ["#_/Directory"]
- },
- "jsonldPredicate": {
- "_id": "@type",
- "_type": "@vocab"
- }
- }, {
- "name": "location",
- "type": "string",
- "jsonldPredicate": "_:location"
- }, {
- "name": "listing",
- "type": {
- "type": "array",
- "items": ["File", "Directory"]
- }
- }],
- }]})
-
- def test_validate_big(self):
- document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(self.schema)
-
- with open("biglisting.yml") as f:
- biglisting = json.load(f)
-
- self.assertEquals(True, validate_ex(avsc_names.get_name("Directory", ""), biglisting,
- strict=True, raise_ex=False))
-
-
- # def test_validate_small(self):
- # document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(self.schema)
-
- # with open("smalllisting.yml") as f:
- # smalllisting = json.load(f)
-
- # validate_ex(avsc_names.get_name("Directory", ""), smalllisting,
- # strict=True, raise_ex=True)
diff --git a/schema_salad/tests/test_validate.py~ b/schema_salad/tests/test_validate.py~
deleted file mode 100644
index db0fd1b..0000000
--- a/schema_salad/tests/test_validate.py~
+++ /dev/null
@@ -1,70 +0,0 @@
-import unittest
-import json
-from schema_salad.schema import load_schema
-from schema_salad.validate import validate_ex
-
-class TestValidate(unittest.TestCase):
- schema = {"name": "_", "$graph":[{
- "name": "File",
- "type": "record",
- "fields": [{
- "name": "class",
- "type": {
- "type": "enum",
- "name": "File_class",
- "symbols": ["#_/File"]
- },
- "jsonldPredicate": {
- "_id": "@type",
- "_type": "@vocab"
- }
- }, {
- "name": "location",
- "type": "string",
- "jsonldPredicate": "_:location"
- }]
- }, {
- "name": "Directory",
- "type": "record",
- "fields": [{
- "name": "class",
- "type": {
- "type": "enum",
- "name": "Directory_class",
- "symbols": ["#_/Directory"]
- },
- "jsonldPredicate": {
- "_id": "@type",
- "_type": "@vocab"
- }
- }, {
- "name": "location",
- "type": "string",
- "jsonldPredicate": "_:location"
- }, {
- "name": "listing",
- "type": {
- "type": "array",
- "items": ["File", "Directory"]
- }
- }],
- }]}
-
- def test_validate_big(self):
- document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(self.schema)
-
- with open("biglisting.yml") as f:
- biglisting = json.load(f)
-
- self.assertEquals(True, validate_ex(avsc_names.get_name("Directory", ""), biglisting,
- strict=True, raise_ex=False))
-
-
- # def test_validate_small(self):
- # document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(self.schema)
-
- # with open("smalllisting.yml") as f:
- # smalllisting = json.load(f)
-
- # validate_ex(avsc_names.get_name("Directory", ""), smalllisting,
- # strict=True, raise_ex=True)
diff --git a/schema_salad/tests/util.py b/schema_salad/tests/util.py
index 0fcaf52..0f71ece 100644
--- a/schema_salad/tests/util.py
+++ b/schema_salad/tests/util.py
@@ -1,7 +1,9 @@
+from __future__ import absolute_import
from pkg_resources import Requirement, resource_filename, ResolutionError # type: ignore
+from typing import Optional, Text
import os
-def get_data(filename):
+def get_data(filename): # type: (Text) -> Optional[Text]
filepath = None
try:
filepath = resource_filename(
diff --git a/schema_salad/flatten.py b/schema_salad/utils.py
similarity index 55%
rename from schema_salad/flatten.py
rename to schema_salad/utils.py
index a417b34..2ba98dc 100644
--- a/schema_salad/flatten.py
+++ b/schema_salad/utils.py
@@ -1,8 +1,23 @@
-import sys
-from typing import Any, Tuple
+from __future__ import absolute_import
+
+from typing import Any, Dict, List
+
+
+def add_dictlist(di, key, val): # type: (Dict, Any, Any) -> None
+ if key not in di:
+ di[key] = []
+ di[key].append(val)
-# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
+def aslist(l): # type: (Any) -> List
+ """Convenience function to wrap single items and lists, and return lists unchanged."""
+
+ if isinstance(l, list):
+ return l
+ else:
+ return [l]
+
+# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
def flatten(l, ltypes=(list, tuple)):
# type: (Any, Any) -> Any
diff --git a/schema_salad/validate.py b/schema_salad/validate.py
index 75e094b..2fab415 100644
--- a/schema_salad/validate.py
+++ b/schema_salad/validate.py
@@ -1,12 +1,19 @@
+from __future__ import absolute_import
import pprint
import avro.schema
from avro.schema import Schema
import sys
-import urlparse
import re
-from typing import Any, Union
+import logging
+
+import six
+from six.moves import urllib
+from six.moves import range
+
+from typing import Any, List, Set, Union, Text
from .sourceline import SourceLine, lineno_re, bullets, indent
+_logger = logging.getLogger("salad")
class ValidationException(Exception):
pass
@@ -18,9 +25,9 @@ class ClassValidationException(ValidationException):
def validate(expected_schema, # type: Schema
datum, # type: Any
- identifiers=set(), # type: Set[unicode]
+ identifiers=[], # type: List[Text]
strict=False, # type: bool
- foreign_properties=set() # type: Set[unicode]
+ foreign_properties=set() # type: Set[Text]
):
# type: (...) -> bool
return validate_ex(
@@ -54,18 +61,20 @@ def vpformat(datum): # type: (Any) -> str
return a
-def validate_ex(expected_schema, # type: Schema
- datum, # type: Any
- identifiers=None, # type: Set[unicode]
- strict=False, # type: bool
- foreign_properties=None, # type: Set[unicode]
- raise_ex=True # type: bool
+def validate_ex(expected_schema, # type: Schema
+ datum, # type: Any
+ identifiers=None, # type: List[Text]
+ strict=False, # type: bool
+ foreign_properties=None, # type: Set[Text]
+ raise_ex=True, # type: bool
+ strict_foreign_properties=False, # type: bool
+ logger=_logger # type: logging.Logger
):
# type: (...) -> bool
"""Determine if a python datum is an instance of a schema."""
if not identifiers:
- identifiers = set()
+ identifiers = []
if not foreign_properties:
foreign_properties = set()
@@ -89,7 +98,7 @@ def validate_ex(expected_schema, # type: Schema
else:
return False
elif schema_type == 'string':
- if isinstance(datum, basestring):
+ if isinstance(datum, six.string_types):
return True
elif isinstance(datum, bytes):
datum = datum.decode(u"utf-8")
@@ -109,7 +118,7 @@ def validate_ex(expected_schema, # type: Schema
else:
return False
elif schema_type == 'int':
- if ((isinstance(datum, int) or isinstance(datum, long))
+ if (isinstance(datum, six.integer_types)
and INT_MIN_VALUE <= datum <= INT_MAX_VALUE):
return True
else:
@@ -118,7 +127,7 @@ def validate_ex(expected_schema, # type: Schema
else:
return False
elif schema_type == 'long':
- if ((isinstance(datum, int) or isinstance(datum, long))
+ if ((isinstance(datum, six.integer_types))
and LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE):
return True
else:
@@ -128,7 +137,7 @@ def validate_ex(expected_schema, # type: Schema
else:
return False
elif schema_type in ['float', 'double']:
- if (isinstance(datum, int) or isinstance(datum, long)
+ if (isinstance(datum, six.integer_types)
or isinstance(datum, float)):
return True
else:
@@ -146,7 +155,7 @@ def validate_ex(expected_schema, # type: Schema
raise ValidationException(u"'Any' type must be non-null")
else:
return False
- if not isinstance(datum, basestring):
+ if not isinstance(datum, six.string_types):
if raise_ex:
raise ValidationException(
u"value is a %s but expected a string" % (type(datum).__name__))
@@ -167,39 +176,44 @@ def validate_ex(expected_schema, # type: Schema
for i, d in enumerate(datum):
try:
sl = SourceLine(datum, i, ValidationException)
- if not validate_ex(expected_schema.items, d, identifiers, strict=strict,
+ if not validate_ex(expected_schema.items, d, identifiers,
+ strict=strict,
foreign_properties=foreign_properties,
- raise_ex=raise_ex):
+ raise_ex=raise_ex,
+ strict_foreign_properties=strict_foreign_properties,
+ logger=logger):
return False
except ValidationException as v:
if raise_ex:
raise sl.makeError(
- unicode("item is invalid because\n%s" % (indent(str(v)))))
+ six.text_type("item is invalid because\n%s" % (indent(str(v)))))
else:
return False
return True
else:
if raise_ex:
- raise ValidationException(u"the value is not a list, expected list of %s" % (
- friendly(expected_schema.items)))
+ raise ValidationException(u"the value %s is not a list, expected list of %s" % (
+ vpformat(datum), friendly(expected_schema.items)))
else:
return False
elif isinstance(expected_schema, avro.schema.UnionSchema):
for s in expected_schema.schemas:
- if validate_ex(s, datum, identifiers, strict=strict, raise_ex=False):
+ if validate_ex(s, datum, identifiers, strict=strict, raise_ex=False,
+ strict_foreign_properties=strict_foreign_properties,
+ logger=logger):
return True
if not raise_ex:
return False
- errors = [] # type: List[unicode]
+ errors = [] # type: List[Text]
checked = []
for s in expected_schema.schemas:
if isinstance(datum, list) and not isinstance(s, avro.schema.ArraySchema):
continue
elif isinstance(datum, dict) and not isinstance(s, avro.schema.RecordSchema):
continue
- elif isinstance(datum, (bool, int, long, float, basestring)) and isinstance(s, (avro.schema.ArraySchema, avro.schema.RecordSchema)):
+ elif isinstance(datum, (bool, six.integer_types, float, six.string_types)) and isinstance(s, (avro.schema.ArraySchema, avro.schema.RecordSchema)):
continue
elif datum is not None and s.type == "null":
continue
@@ -207,12 +221,15 @@ def validate_ex(expected_schema, # type: Schema
checked.append(s)
try:
validate_ex(s, datum, identifiers, strict=strict,
- foreign_properties=foreign_properties, raise_ex=True)
+ foreign_properties=foreign_properties,
+ raise_ex=True,
+ strict_foreign_properties=strict_foreign_properties,
+ logger=logger)
except ClassValidationException as e:
raise
except ValidationException as e:
- errors.append(unicode(e))
- if errors:
+ errors.append(six.text_type(e))
+ if bool(errors):
raise ValidationException(bullets(["tried %s but\n%s" % (friendly(
checked[i]), indent(errors[i])) for i in range(0, len(errors))], "- "))
else:
@@ -237,7 +254,11 @@ def validate_ex(expected_schema, # type: Schema
else:
return False
if expected_schema.name != d:
- return False
+ if raise_ex:
+ raise ValidationException(
+ u"Expected class '%s' but this is '%s'" % (expected_schema.name, d))
+ else:
+ return False
classmatch = d
break
@@ -255,9 +276,12 @@ def validate_ex(expected_schema, # type: Schema
fieldval = None
try:
- sl = SourceLine(datum, f.name, unicode)
- if not validate_ex(f.type, fieldval, identifiers, strict=strict, foreign_properties=foreign_properties,
- raise_ex=raise_ex):
+ sl = SourceLine(datum, f.name, six.text_type)
+ if not validate_ex(f.type, fieldval, identifiers, strict=strict,
+ foreign_properties=foreign_properties,
+ raise_ex=raise_ex,
+ strict_foreign_properties=strict_foreign_properties,
+ logger=logger):
return False
except ValidationException as v:
if f.name not in datum:
@@ -266,26 +290,36 @@ def validate_ex(expected_schema, # type: Schema
errors.append(sl.makeError(u"the `%s` field is not valid because\n%s" % (
f.name, indent(str(v)))))
- if strict:
- for d in datum:
- found = False
- for f in expected_schema.fields:
- if d == f.name:
- found = True
- if not found:
- sl = SourceLine(datum, d, unicode)
- if d not in identifiers and d not in foreign_properties and d[0] not in ("@", "$"):
- if not raise_ex:
- return False
- split = urlparse.urlsplit(d)
- if split.scheme:
- errors.append(sl.makeError(
- u"unrecognized extension field `%s` and strict is True. Did you include a $schemas section?" % (d)))
+ for d in datum:
+ found = False
+ for f in expected_schema.fields:
+ if d == f.name:
+ found = True
+ if not found:
+ sl = SourceLine(datum, d, six.text_type)
+ if d not in identifiers and d not in foreign_properties and d[0] not in ("@", "$"):
+ if (d not in identifiers and strict) and (
+ d not in foreign_properties and strict_foreign_properties) and not raise_ex:
+ return False
+ split = urllib.parse.urlsplit(d)
+ if split.scheme:
+ err = sl.makeError(u"unrecognized extension field `%s`%s."
+ " Did you include "
+ "a $schemas section?" % (
+ d, " and strict_foreign_properties is True" if strict_foreign_properties else ""))
+ if strict_foreign_properties:
+ errors.append(err)
+ else:
+ logger.warn(err)
+ else:
+ err = sl.makeError(u"invalid field `%s`, expected one of: %s" % (
+ d, ", ".join("'%s'" % fn.name for fn in expected_schema.fields)))
+ if strict:
+ errors.append(err)
else:
- errors.append(sl.makeError(u"invalid field `%s`, expected one of: %s" % (
- d, ", ".join("'%s'" % fn.name for fn in expected_schema.fields))))
+ logger.warn(err)
- if errors:
+ if bool(errors):
if raise_ex:
if classmatch:
raise ClassValidationException(bullets(errors, "* "))
diff --git a/setup.cfg b/setup.cfg
index 5bc692a..bab3a64 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -11,7 +11,6 @@ test = pytest
addopts = --pyarg schema_salad
[egg_info]
-tag_build = .20170119151016
+tag_build = .20170630075932
tag_date = 0
-tag_svn_revision = 0
diff --git a/setup.py b/setup.py
index 1e8c2fd..80192c1 100755
--- a/setup.py
+++ b/setup.py
@@ -30,24 +30,21 @@ else:
install_requires = [
'setuptools',
'requests >= 1.0',
- 'ruamel.yaml >= 0.12.4',
- 'rdflib >= 4.2.0, < 4.3.0',
+ 'ruamel.yaml >= 0.12.4, < 0.15',
+ 'rdflib >= 4.2.2, < 4.3.0',
'rdflib-jsonld >= 0.3.0, < 0.5.0',
'mistune >= 0.7.3, < 0.8',
- 'typing >= 3.5.2, < 3.6',
+ 'typing >= 3.5.3',
'CacheControl >= 0.11.7, < 0.12',
- 'lockfile >= 0.9']
+ 'lockfile >= 0.9',
+ 'six >= 1.8.0']
-install_requires.append("avro") # TODO: remove me once cwltool is
-# available in Debian Stable, Ubuntu 12.04 LTS
-
-# extras_require={ # TODO: uncomment me, same conditions as above
-# ':python_version<"3"': ['avro'],
-# ':python_version>="3"': ['avro-python3']}
-extras_require = {} # TODO: to be removed when the above is added
+extras_require={
+ ':python_version<"3"': ['avro'],
+ ':python_version>="3"': ['avro-python3']}
setup(name='schema-salad',
- version='2.2',
+ version='2.6',
description='Schema Annotations for Linked Avro Data (SALAD)',
long_description=open(README).read(),
author='Common workflow language working group',
@@ -64,7 +61,7 @@ setup(name='schema-salad',
test_suite='tests',
tests_require=['pytest'],
entry_points={
- 'console_scripts': ["schema-salad-tool=schema_salad.main:main"]
+ 'console_scripts': ["schema-salad-tool=schema_salad.main:main", "schema-salad-doc=schema_salad.makedoc:main"]
},
zip_safe=True,
cmdclass={'egg_info': tagger},
@@ -75,8 +72,9 @@ setup(name='schema-salad',
"Operating System :: MacOS :: MacOS X",
"Development Status :: 4 - Beta",
"Programming Language :: Python :: 2.7",
- #"Programming Language :: Python :: 3.3", # TODO: uncomment these
- #"Programming Language :: Python :: 3.4", # lines
- #"Programming Language :: Python :: 3.5"
+ "Programming Language :: Python :: 3.3",
+ "Programming Language :: Python :: 3.4",
+ "Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6"
]
)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-schema-salad.git
More information about the debian-med-commit
mailing list