#!/usr/bin/env python
"""
Extraction parsers for structured data embedded into HTML or XML files.
The former may include RDFa or microdata. The syntax and the extraction
procedures are based on:
* The RDFa specifications: http://www.w3.org/TR/#tr_RDFa
* The microdata specification: http://www.w3.org/TR/microdata/
* The specification of the microdata to RDF conversion:
http://www.w3.org/TR/microdata-rdf/
License: W3C Software License,
http://www.w3.org/Consortium/Legal/copyright-software
Author: Ivan Herman
Copyright: W3C
"""
from rdflib.parser import (
Parser, StringInputSource, URLInputSource, FileInputSource)
try:
import html5lib
assert html5lib
html5lib = True
except ImportError:
import warnings
warnings.warn(
'html5lib not found! RDFa and Microdata ' +
'parsers will not be available.')
html5lib = False
def _get_orig_source(source):
"""
A bit of a hack; the RDFa/microdata parsers need more than what the
upper layers of RDFLib provide...
This method returns the original source references.
"""
if isinstance(source, StringInputSource):
orig_source = source.getByteStream()
elif isinstance(source, URLInputSource):
orig_source = source.url
elif isinstance(source, FileInputSource):
orig_source = source.file.name
source.file.close()
else:
orig_source = source.getByteStream()
baseURI = source.getPublicId()
return (baseURI, orig_source)
def _check_error(graph):
from .pyRdfa import RDFA_Error, ns_rdf
from .pyRdfa.options import ns_dc
for (s, p, o) in graph.triples((None, ns_rdf["type"], RDFA_Error)):
for (x, y, msg) in graph.triples((s, ns_dc["description"], None)):
raise Exception("RDFa parsing Error! %s" % msg)
# This is the parser interface as it would look when called from the
# rest of RDFLib
[docs]class RDFaParser(Parser):
"""
Wrapper around the RDFa 1.1 parser. For further details on the RDFa 1.1
processing, see the relevant W3C documents at
http://www.w3.org/TR/#tr_RDFa. RDFa 1.1 is defined for XHTML, HTML5, SVG
and, in general, for any XML language.
Note that the parser can also handle RDFa 1.0 if the extra parameter is
used and/or the input source uses RDFa 1.0 specific @version or DTD-s.
"""
[docs] def parse(self, source, graph,
pgraph=None,
media_type="",
rdfa_version=None,
embedded_rdf=False,
space_preserve=True,
vocab_expansion=False,
vocab_cache=False,
refresh_vocab_cache=False,
vocab_cache_report=False,
check_lite=False):
"""
@param source: one of the input sources that the RDFLib package defined
@type source: InputSource class instance
@param graph: target graph for the triples; output graph, in RDFa spec.
parlance
@type graph: RDFLib Graph
@keyword pgraph: target for error and warning triples; processor graph,
in RDFa spec. parlance. If set to None, these triples are ignored
@type pgraph: RDFLib Graph
@keyword media_type: explicit setting of the preferred media type
(a.k.a. content type) of the the RDFa source. None means the content
type of the HTTP result is used, or a guess is made based on the
suffix of a file
@type media_type: string
@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by
default, 1.1 is used unless the source has explicit signals to use
1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
@type rdfa_version: string
@keyword embedded_rdf: some formats allow embedding RDF in other
formats: (X)HTML can contain turtle in a special <script> element,
SVG can have RDF/XML embedded in a <metadata> element. This flag
controls whether those triples should be interpreted and added to
the output graph. Some languages (e.g., SVG) require this, and the
flag is ignored.
@type embedded_rdf: Boolean
@keyword space_preserve: by default, space in the HTML source must be preserved in the generated literal;
this behavior can be switched off
@type space_preserve: Boolean
@keyword vocab_expansion: whether the RDFa @vocab attribute should
also mean vocabulary expansion (see the RDFa 1.1 spec for further
details)
@type vocab_expansion: Boolean
@keyword vocab_cache: in case vocab expansion is used, whether the
expansion data (i.e., vocabulary) should be cached locally. This
requires the ability for the local application to write on the
local file system
@type vocab_chache: Boolean
@keyword vocab_cache_report: whether the details of vocabulary file caching process should be reported
in the processor graph as information (mainly useful for debug)
@type vocab_cache_report: Boolean
@keyword refresh_vocab_cache: whether the caching checks of vocabs should be by-passed, ie, if caches should be re-generated regardless of the stored date (important for vocab development)
@type refresh_vocab_cache: Boolean
@keyword check_lite: generate extra warnings in case the input source is not RDFa 1.1 check_lite
@type check_lite: Boolean
"""
if html5lib is False:
raise ImportError(
'html5lib is not installed, cannot use ' +
'RDFa and Microdata parsers.')
(baseURI, orig_source) = _get_orig_source(source)
self._process(graph, pgraph, baseURI, orig_source,
media_type=media_type,
rdfa_version=rdfa_version,
embedded_rdf=embedded_rdf,
space_preserve=space_preserve,
vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache,
vocab_cache_report=vocab_cache_report,
refresh_vocab_cache=refresh_vocab_cache,
check_lite=check_lite
)
def _process(self, graph, pgraph, baseURI, orig_source,
media_type="",
rdfa_version=None,
embedded_rdf=False,
space_preserve=True,
vocab_expansion=False,
vocab_cache=False,
vocab_cache_report=False,
refresh_vocab_cache=False,
check_lite=False):
from .pyRdfa import pyRdfa, Options
from rdflib import Graph
processor_graph = pgraph if pgraph is not None else Graph()
self.options = Options(output_processor_graph=True,
embedded_rdf=embedded_rdf,
space_preserve=space_preserve,
vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache,
vocab_cache_report=vocab_cache_report,
refresh_vocab_cache=refresh_vocab_cache,
check_lite=check_lite)
if media_type is None:
media_type = ""
processor = pyRdfa(self.options,
base=baseURI,
media_type=media_type,
rdfa_version=rdfa_version)
processor.graph_from_source(orig_source, graph=graph, pgraph=processor_graph, rdfOutput=False)
# This may result in an exception if the graph parsing led to an error
_check_error(processor_graph)
[docs]class RDFa10Parser(Parser):
"""
This is just a convenience class to wrap around the RDFa 1.0 parser.
"""
[docs] def parse(self, source, graph, pgraph=None, media_type=""):
"""
@param source: one of the input sources that the RDFLib package defined
@type source: InputSource class instance
@param graph: target graph for the triples; output graph, in RDFa
spec. parlance
@type graph: RDFLib Graph
@keyword pgraph: target for error and warning triples; processor
graph, in RDFa spec. parlance. If set to None, these triples are
ignored
@type pgraph: RDFLib Graph
@keyword media_type: explicit setting of the preferred media type
(a.k.a. content type) of the the RDFa source. None means the content
type of the HTTP result is used, or a guess is made based on the
suffix of a file
@type media_type: string
@keyword rdfOutput: whether Exceptions should be catched and added,
as triples, to the processor graph, or whether they should be raised.
@type rdfOutput: Boolean
"""
RDFaParser().parse(source, graph, pgraph=pgraph,
media_type=media_type, rdfa_version="1.0")
[docs]class MicrodataParser(Parser):
"""
Wrapper around an HTML5 microdata, extracted and converted into RDF. For
the specification of microdata, see the relevant section of the HTML5
spec: http://www.w3.org/TR/microdata/; for the algorithm used to extract
microdata into RDF, see http://www.w3.org/TR/microdata-rdf/.
"""
[docs] def parse(self, source, graph, vocab_expansion=False, vocab_cache=False):
"""
@param source: one of the input sources that the RDFLib package defined
@type source: InputSource class instance
@param graph: target graph for the triples; output graph, in RDFa
spec. parlance
@type graph: RDFLib Graph
@keyword vocab_expansion: whether the RDFa @vocab attribute should
also mean vocabulary expansion (see the RDFa 1.1 spec for further
details)
@type vocab_expansion: Boolean
@keyword vocab_cache: in case vocab expansion is used, whether the
expansion data (i.e., vocabulary) should be cached locally. This
requires the ability for the local application to write on the
local file system
@type vocab_chache: Boolean
@keyword rdfOutput: whether Exceptions should be catched and added,
as triples, to the processor graph, or whether they should be raised.
@type rdfOutput: Boolean
"""
if html5lib is False:
raise ImportError(
'html5lib is not installed, cannot use RDFa ' +
'and Microdata parsers.')
(baseURI, orig_source) = _get_orig_source(source)
self._process(graph, baseURI, orig_source,
vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache)
def _process(self, graph, baseURI, orig_source,
vocab_expansion=False, vocab_cache=False):
from .pyMicrodata import pyMicrodata
processor = pyMicrodata(base=baseURI, vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache)
processor.graph_from_source(
orig_source, graph=graph, rdfOutput=False)
[docs]class StructuredDataParser(Parser):
"""
Convenience parser to extract both RDFa (including embedded Turtle)
and microdata from an HTML file.
It is simply a wrapper around the specific parsers.
"""
[docs] def parse(self, source, graph,
pgraph=None,
rdfa_version="",
vocab_expansion=False,
vocab_cache=False,
media_type='text/html'
):
"""
@param source: one of the input sources that the RDFLib package defined
@type source: InputSource class instance
@param graph: target graph for the triples; output graph, in RDFa
spec. parlance
@keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by
default, 1.1 is used unless the source has explicit signals to use 1.0
(e.g., using a @version attribute, using a DTD set up for 1.0, etc)
@type rdfa_version: string
@type graph: RDFLib Graph
@keyword pgraph: target for error and warning triples; processor
graph, in RDFa spec. parlance. If set to None, these triples are
ignored
@type pgraph: RDFLib Graph
@keyword vocab_expansion: whether the RDFa @vocab attribute should
also mean vocabulary expansion (see the RDFa 1.1 spec for further
details)
@type vocab_expansion: Boolean
@keyword vocab_cache: in case vocab expansion is used, whether the
expansion data (i.e., vocabulary) should be cached locally. This
requires the ability for the local application to write on the
local file system
@type vocab_chache: Boolean
@keyword rdfOutput: whether Exceptions should be catched and added,
as triples, to the processor graph, or whether they should be raised.
@type rdfOutput: Boolean
"""
# Note that the media_type argument is ignored, and is here only to avoid an 'unexpected argument' error.
# This parser works for text/html only anyway...
(baseURI, orig_source) = _get_orig_source(source)
if rdfa_version == "" : rdfa_version = "1.1"
RDFaParser()._process(graph, pgraph, baseURI, orig_source,
media_type='text/html',
rdfa_version=rdfa_version,
vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache)
MicrodataParser()._process(graph, baseURI, orig_source,
vocab_expansion=vocab_expansion,
vocab_cache=vocab_cache)
from .hturtle import HTurtleParser
HTurtleParser()._process(graph, baseURI, orig_source, media_type='text/html')
