Use Cython if available; added Python 3 support to _yaml.pyx.
diff --git a/ext/_yaml.h b/ext/_yaml.h
index d8070e5..b39292d 100644
--- a/ext/_yaml.h
+++ b/ext/_yaml.h
@@ -1,3 +1,11 @@
#include <yaml.h>
+#if PY_MAJOR_VERSION >= 3
+
+#define PyString_CheckExact PyBytes_CheckExact
+#define PyString_AS_STRING PyBytes_AS_STRING
+#define PyString_GET_SIZE PyBytes_GET_SIZE
+#define PyString_FromStringAndSize PyBytes_FromStringAndSize
+
+#endif
diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx
index b2c4e52..8b11f16 100644
--- a/ext/_yaml.pyx
+++ b/ext/_yaml.pyx
@@ -251,10 +251,16 @@
cdef object anchors
def __init__(self, stream):
+ cdef is_readable
if yaml_parser_initialize(&self.parser) == 0:
raise MemoryError
self.parsed_event.type = YAML_NO_EVENT
- if hasattr(stream, 'read'):
+ is_readable = 1
+ try:
+ stream.read
+ except AttributeError:
+ is_readable = 0
+ if is_readable:
self.stream = stream
try:
self.stream_name = stream.name
@@ -357,23 +363,25 @@
elif token.type == YAML_STREAM_START_TOKEN:
encoding = None
if token.data.stream_start.encoding == YAML_UTF8_ENCODING:
- encoding = "utf-8"
+ encoding = u"utf-8"
elif token.data.stream_start.encoding == YAML_UTF16LE_ENCODING:
- encoding = "utf-16-le"
+ encoding = u"utf-16-le"
elif token.data.stream_start.encoding == YAML_UTF16BE_ENCODING:
- encoding = "utf-16-be"
+ encoding = u"utf-16-be"
return StreamStartToken(start_mark, end_mark, encoding)
elif token.type == YAML_STREAM_END_TOKEN:
return StreamEndToken(start_mark, end_mark)
elif token.type == YAML_VERSION_DIRECTIVE_TOKEN:
- return DirectiveToken("YAML",
+ return DirectiveToken(u"YAML",
(token.data.version_directive.major,
token.data.version_directive.minor),
start_mark, end_mark)
elif token.type == YAML_TAG_DIRECTIVE_TOKEN:
- return DirectiveToken("TAG",
- (token.data.tag_directive.handle,
- token.data.tag_directive.prefix),
+ handle = PyUnicode_DecodeUTF8(token.data.tag_directive.handle,
+ strlen(token.data.tag_directive.handle), 'strict')
+ prefix = PyUnicode_DecodeUTF8(token.data.tag_directive.prefix,
+ strlen(token.data.tag_directive.prefix), 'strict')
+ return DirectiveToken(u"TAG", (handle, prefix),
start_mark, end_mark)
elif token.type == YAML_DOCUMENT_START_TOKEN:
return DocumentStartToken(start_mark, end_mark)
@@ -870,6 +878,8 @@
cdef CParser parser
parser = <CParser>data
value = parser.stream.read(size)
+ if PyUnicode_CheckExact(value) != 0:
+ value = PyUnicode_AsUTF8String(value)
if PyString_CheckExact(value) == 0:
raise TypeError("a string value is expected")
if PyString_GET_SIZE(value) > size:
@@ -894,6 +904,7 @@
cdef object anchors
cdef int last_alias_id
cdef int closed
+ cdef int decode_output
def __init__(self, stream, canonical=None, indent=None, width=None,
allow_unicode=None, line_break=None, encoding=None,
@@ -901,6 +912,11 @@
if yaml_emitter_initialize(&self.emitter) == 0:
raise MemoryError
self.stream = stream
+ self.decode_output = 1
+ try:
+ stream.encoding
+ except AttributeError:
+ self.decode_output = 0
yaml_emitter_set_output(&self.emitter, output_handler, <void *>self)
if canonical is not None:
yaml_emitter_set_canonical(&self.emitter, 1)
@@ -1216,7 +1232,7 @@
if node in self.anchors:
if self.anchors[node] is None:
self.last_alias_id = self.last_alias_id+1
- self.anchors[node] = "id%03d" % self.last_alias_id
+ self.anchors[node] = u"id%03d" % self.last_alias_id
else:
self.anchors[node] = None
node_class = node.__class__
@@ -1245,7 +1261,7 @@
anchor_object = self.anchors[node]
anchor = NULL
if anchor_object is not None:
- anchor = PyString_AS_STRING(anchor_object)
+ anchor = PyString_AS_STRING(PyUnicode_AsUTF8String(anchor_object))
if node in self.serialized_nodes:
if yaml_alias_event_initialize(&event, anchor) == 0:
raise MemoryError
@@ -1357,7 +1373,10 @@
cdef int output_handler(void *data, char *buffer, int size) except 0:
cdef CEmitter emitter
emitter = <CEmitter>data
- value = PyString_FromStringAndSize(buffer, size)
+ if emitter.decode_output == 0:
+ value = PyString_FromStringAndSize(buffer, size)
+ else:
+ value = PyUnicode_DecodeUTF8(buffer, size, 'strict')
emitter.stream.write(value)
return 1
diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py
index b180319..55592e0 100644
--- a/lib3/yaml/__init__.py
+++ b/lib3/yaml/__init__.py
@@ -1,7 +1,4 @@
-__version__ = '3.08'
-__with_libyaml__ = False
-
from .error import *
from .tokens import *
@@ -11,6 +8,13 @@
from .loader import *
from .dumper import *
+__version__ = '3.08'
+try:
+ from .cyaml import *
+ __with_libyaml__ = True
+except ImportError:
+ __with_libyaml__ = False
+
import io
def scan(stream, Loader=Loader):
diff --git a/lib3/yaml/cyaml.py b/lib3/yaml/cyaml.py
new file mode 100644
index 0000000..d5cb87e
--- /dev/null
+++ b/lib3/yaml/cyaml.py
@@ -0,0 +1,85 @@
+
+__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
+ 'CBaseDumper', 'CSafeDumper', 'CDumper']
+
+from _yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
diff --git a/setup.py b/setup.py
index 9d73944..3f396a9 100644
--- a/setup.py
+++ b/setup.py
@@ -73,12 +73,19 @@
sys.modules['distutils.extension'].Extension = _Extension
sys.modules['distutils.command.build_ext'].Extension = _Extension
-try:
- from Pyrex.Distutils import Extension as _Extension
- from Pyrex.Distutils import build_ext as _build_ext
- with_pyrex = True
-except ImportError:
- with_pyrex = False
+with_pyrex = None
+if sys.version_info[0] < 3:
+ try:
+ from Cython.Distutils.extension import Extension as _Extension
+ from Cython.Distutils import build_ext as _build_ext
+ with_pyrex = 'cython'
+ except ImportError:
+ try:
+ from Pyrex.Distutils import Extension as _Extension
+ from Pyrex.Distutils import build_ext as _build_ext
+ with_pyrex = 'pyrex'
+ except ImportError:
+ pass
class Distribution(_Distribution):
@@ -167,8 +174,10 @@
self.check_extensions_list(self.extensions)
filenames = []
for ext in self.extensions:
- if with_pyrex:
+ if with_pyrex == 'pyrex':
self.pyrex_sources(ext.sources, ext)
+ elif with_pyrex == 'cython':
+ self.cython_sources(ext.sources, ext)
for filename in ext.sources:
filenames.append(filename)
base = os.path.splitext(filename)[0]
@@ -197,8 +206,10 @@
with_ext = self.check_extension_availability(ext)
if not with_ext:
continue
- if with_pyrex:
+ if with_pyrex == 'pyrex':
ext.sources = self.pyrex_sources(ext.sources, ext)
+ elif with_pyrex == 'cython':
+ ext.sources = self.cython_sources(ext.sources, ext)
self.build_extension(ext)
def check_extension_availability(self, ext):
@@ -297,57 +308,37 @@
if __name__ == '__main__':
- if sys.version_info[0] < 3:
+ package_dir = {
+ '2': 'lib',
+ }
- setup(
- name=NAME,
- version=VERSION,
- description=DESCRIPTION,
- long_description=LONG_DESCRIPTION,
- author=AUTHOR,
- author_email=AUTHOR_EMAIL,
- license=LICENSE,
- platforms=PLATFORMS,
- url=URL,
- download_url=DOWNLOAD_URL,
- classifiers=CLASSIFIERS,
+ setup(
+ name=NAME,
+ version=VERSION,
+ description=DESCRIPTION,
+ long_description=LONG_DESCRIPTION,
+ author=AUTHOR,
+ author_email=AUTHOR_EMAIL,
+ license=LICENSE,
+ platforms=PLATFORMS,
+ url=URL,
+ download_url=DOWNLOAD_URL,
+ classifiers=CLASSIFIERS,
- package_dir={'': 'lib'},
- packages=['yaml'],
- ext_modules=[
- Extension('_yaml', ['ext/_yaml.pyx'],
- 'libyaml', "LibYAML bindings", LIBYAML_CHECK,
- libraries=['yaml']),
- ],
+ package_dir={'': {2: 'lib', 3: 'lib3'}[sys.version_info[0]]},
+ packages=['yaml'],
+ ext_modules=[
+ Extension('_yaml', ['ext/_yaml.pyx'],
+ 'libyaml', "LibYAML bindings", LIBYAML_CHECK,
+ libraries=['yaml']),
+ ],
- distclass=Distribution,
- cmdclass={
- 'build_ext': build_ext,
- 'bdist_rpm': bdist_rpm,
- 'test': test,
- },
- )
+ distclass=Distribution,
- else:
-
- setup(
- name=NAME,
- version=VERSION,
- description=DESCRIPTION,
- long_description=LONG_DESCRIPTION,
- author=AUTHOR,
- author_email=AUTHOR_EMAIL,
- license=LICENSE,
- platforms=PLATFORMS,
- url=URL,
- download_url=DOWNLOAD_URL,
- classifiers=CLASSIFIERS,
-
- package_dir={'': 'lib3'},
- packages=['yaml'],
-
- cmdclass={
- 'test': test,
- },
- )
+ cmdclass={
+ 'build_ext': build_ext,
+ 'bdist_rpm': bdist_rpm,
+ 'test': test,
+ },
+ )
diff --git a/tests/lib3/test_structure.py b/tests/lib3/test_structure.py
index 4d9c4ea..6d6f59d 100644
--- a/tests/lib3/test_structure.py
+++ b/tests/lib3/test_structure.py
@@ -139,7 +139,7 @@
return tuple(yaml.Loader.construct_sequence(self, node))
def construct_mapping(self, node):
pairs = self.construct_pairs(node)
- pairs.sort()
+ pairs.sort(key=(lambda i: str(i)))
return pairs
def construct_undefined(self, node):
return self.construct_scalar(node)
@@ -155,7 +155,7 @@
return tuple(yaml.CanonicalLoader.construct_sequence(self, node))
def construct_mapping(self, node):
pairs = self.construct_pairs(node)
- pairs.sort()
+ pairs.sort(key=(lambda i: str(i)))
return pairs
def construct_undefined(self, node):
return self.construct_scalar(node)
diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py
index 57c02c6..8e36e5d 100644
--- a/tests/lib3/test_yaml_ext.py
+++ b/tests/lib3/test_yaml_ext.py
@@ -252,15 +252,13 @@
for collection in collections:
if not isinstance(collection, dict):
collection = vars(collection)
- keys = collection.keys()
- keys.sort()
- for key in keys:
+ for key in sorted(collection):
value = collection[key]
if isinstance(value, types.FunctionType) and hasattr(value, 'unittest'):
functions.append(wrap_ext_function(value))
for function in functions:
- assert function.unittest_name not in globals()
- globals()[function.unittest_name] = function
+ assert function.__name__ not in globals()
+ globals()[function.__name__] = function
import test_tokens, test_structure, test_errors, test_resolver, test_constructor, \
test_emitter, test_representer, test_recursive
