Prepare work before release beta version.

.gitmodules

@@ -2,3 +2,7 @@
 	path = vectors
 	url = https://github.com/amzn/ion-tests
 	branch = master
+[submodule "ion-c"]
+	path = ion-c
+	url = https://github.com/amzn/ion-c
+	branch = master

amazon/ion/ioncmodule.c

@@ -13,7 +13,7 @@
 #define MICROSECOND_DIGITS 6
 
 #define ERR_MSG_MAX_LEN 100
-#define FIELD_NAME_MAX_LEN 100
+#define FIELD_NAME_MAX_LEN 1000
 
 static char _err_msg[ERR_MSG_MAX_LEN];
 
@@ -667,11 +667,11 @@ iERR ionc_write_value(hWRITER writer, PyObject* obj, PyObject* tuple_as_sexp) {
         }
 
         // Determine lsu.
-        int lsu_array[digits_len];
-        for (int i=0; i<digits_len; i++) {
+        int c_digits_array[digits_len];
+        for (int i = 0; i < digits_len; i++) {
             PyObject* digit = PyTuple_GetItem(py_digits, i);
             Py_INCREF(digit);
-            lsu_array[i] = PyLong_AsLong(digit);
+            c_digits_array[i] = PyLong_AsLong(digit);
             Py_DECREF(digit);
         }
         Py_XDECREF(py_digits);
@@ -682,7 +682,7 @@ iERR ionc_write_value(hWRITER writer, PyObject* obj, PyObject* tuple_as_sexp) {
             decNumberUnit per_digit = 0;
             int op_count = count + 1 < DECDPUN ? count + 1 : DECDPUN;
             for (int i = 0; i < op_count; i++) {
-                per_digit += pow(10, i) * lsu_array[count--];
+                per_digit += pow(10, i) * c_digits_array[count--];
             }
             decNumber_value.lsu[index++] = per_digit;
         }
@@ -998,7 +998,7 @@ static iERR ionc_read_timestamp(hREADER hreader, PyObject** timestamp_out) {
             decQuadScaleB(&fraction, &fraction, decQuadFromInt32(&tmp, MICROSECOND_DIGITS), &dec_context);
             int32_t microsecond = decQuadToInt32Exact(&fraction, &dec_context, DEC_ROUND_DOWN);
             if (fractional_precision > MICROSECOND_DIGITS) {
-                // Python only supports up to microsecond precision
+                // C extension only supports up to microsecond precision.
                 fractional_precision = MICROSECOND_DIGITS;
             }
 
@@ -1078,6 +1078,10 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
     if (in_struct) {
         IONCHECK(ion_reader_get_field_name(hreader, &field_name));
         field_name_len = field_name.length;
+        if (field_name_len > FIELD_NAME_MAX_LEN) {
+            _FAILWITHMSG(IERR_INVALID_ARG,
+                "Filed name overflow, please try again with pure python.");
+        }
         if (field_name.value != NULL) {
             None_field_name = FALSE;
             strcpy(field_name_value, field_name.value);
@@ -1312,21 +1316,25 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
         default:
             FAILWITH(IERR_INVALID_STATE);
         }
+
+    PyObject* temp = py_value;
     if (!emit_bare_values) {
-        py_value = PyObject_CallFunctionObjArgs(
+        temp = PyObject_CallFunctionObjArgs(
             ion_nature_constructor,
             py_ion_type_table[ion_type >> 8],
             py_value,
             py_annotations,
             NULL
         );
+        Py_XDECREF(py_value);
+        Py_XDECREF(py_annotations);
     }
 
     if (in_struct && !None_field_name) {
         ION_STRING_INIT(&field_name);
         ion_string_assign_cstr(&field_name, field_name_value, field_name_len);
     }
-    ionc_add_to_container(container, py_value, in_struct, &field_name);
+    ionc_add_to_container(container, temp, in_struct, &field_name);
 
 fail:
     if (err) {

install.py

@@ -30,9 +30,14 @@
 _MAC = _OS == 'Darwin'
 _LINUX = _OS == 'Linux'
 
-_C_EXT_DEPENDENCY_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c-build'))
+_C_EXT_DEPENDENCY_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'amazon/ion/ion-c-build'))
 _C_EXT_DEPENDENCY_LIB_LOCATION = abspath(join(_C_EXT_DEPENDENCY_DIR, 'lib'))
-_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS = abspath(join(_C_EXT_DEPENDENCY_DIR, 'include'))
+_C_EXT_DEPENDENCY_INCLUDES_DIR = abspath(join(_C_EXT_DEPENDENCY_DIR, 'include'))
+_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS = {
+    'ionc': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'ionc')),
+    'decNumber': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'decNumber'))
+}
+_CURRENT_ION_C_DIR = './ion-c'
 
 _IONC_REPO_URL = "https://github.com/amzn/ion-c.git"
 _IONC_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c'))
@@ -63,21 +68,17 @@ def _library_exists():
 
 
 def _library_exists_helper(name):
-    return os.path.exists(join(_C_EXT_DEPENDENCY_LIB_LOCATION, _get_lib_name(name)))
+    return os.path.exists(join(_IONC_INCLUDES_LOCATIONS[name])) \
+        and os.path.exists(join(_IONC_LOCATION, name))
 
 
 def _download_ionc():
     try:
-        # Create a directory to store build output.
-        if not isdir(_C_EXT_DEPENDENCY_DIR):
-            os.mkdir(_C_EXT_DEPENDENCY_DIR)
-            os.mkdir(_C_EXT_DEPENDENCY_LIB_LOCATION)
-            os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS)
-
         # Install ion-c.
-        if not isdir('./ion-c'):
-            check_call(['git', 'clone', '--recurse-submodules', _IONC_REPO_URL, 'ion-c'])
-        os.chdir('ion-c/')
+        if isdir(_CURRENT_ION_C_DIR):
+            shutil.rmtree(_CURRENT_ION_C_DIR)
+        check_call(['git', 'submodule', 'update', '--init', '--recursive'])
+        os.chdir(_CURRENT_ION_C_DIR)
 
         # Initialize submodule.
         check_call(['git', 'submodule', 'update', '--init'])
@@ -86,21 +87,12 @@ def _download_ionc():
         _build_ionc()
 
         os.chdir('../')
-        move_build_lib_for_distribution()
+        return True
     except:
-        if isdir(_C_EXT_DEPENDENCY_DIR):
-            shutil.rmtree(_C_EXT_DEPENDENCY_DIR)
         if isdir(_IONC_DIR):
             shutil.rmtree(_IONC_DIR)
         print('ionc build error: Unable to build ion-c library.')
         return False
-    finally:
-        # Clean up work.
-        temp_ionc_dir = "./ion-c-build"
-        if isdir(temp_ionc_dir):
-            shutil.rmtree(temp_ionc_dir)
-        if isdir(_IONC_DIR):
-            shutil.rmtree(_IONC_DIR)
 
 
 def _build_ionc():
@@ -110,41 +102,44 @@ def _build_ionc():
         _build_ionc_mac_and_linux()
 
 
+def _move_ionc():
+    # move ion-c to output dir.
+    if _WIN:
+        _move_lib_win('ionc')
+        _move_lib_win('decNumber')
+    elif _MAC or _LINUX:
+        _move_lib_mac_and_linux('ionc')
+        _move_lib_mac_and_linux('decNumber')
+
+
 def _build_ionc_win():
     # check_call('cmake -G \"Visual Studio 15 2017 Win64\"')
     check_call('cmake -G \"Visual Studio 16 2019\"')
     check_call('cmake --build . --config Release')
 
-    # move ion-c to output dir.
-    _move_lib_win('ionc')
-    _move_lib_win('decNumber')
-
 
 def _move_lib_win(name):
     """
     Move library and its include files to ion-c-build/lib and ion-c-build/include respectively.
     """
-    shutil.move(_IONC_INCLUDES_LOCATIONS[name], _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS)
+    for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]):
+        shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name])
 
     lib_path = join(_IONC_DIR, name, 'Release', '%s%s' % (name, _LIB_SUFFIX_WIN))
     shutil.copy(lib_path, _C_EXT_DEPENDENCY_LIB_LOCATION)
 
 
-
 def _build_ionc_mac_and_linux():
     # build ion-c.
     check_call(['./build-release.sh'])
 
-    # move ion-c to output dir.
-    _move_lib_mac_and_linux('ionc')
-    _move_lib_mac_and_linux('decNumber')
-
 
 def _move_lib_mac_and_linux(name):
     """
     Move library and its include files to ion-c-build/lib and ion-c-build/include respectively.
     """
-    shutil.move(_IONC_INCLUDES_LOCATIONS[name], _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS)
+    for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]):
+        shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name])
 
     dir_path = join(_IONC_LOCATION, name)
     for file in os.listdir(dir_path):
@@ -158,12 +153,16 @@ def _move_lib_mac_and_linux(name):
 
 
 def move_build_lib_for_distribution():
-    # move ion-c-build to amazon/ion for distribution.
-    target_path = abspath(join(dirname(os.path.abspath(__file__)), 'amazon/ion/ion-c-build'))
-    print('build files are moved to %s.' % target_path)
-    if os.path.isdir(target_path):
-        shutil.rmtree(target_path)
-    shutil.copytree(_C_EXT_DEPENDENCY_DIR, target_path)
+    # Create a directory to store build output.
+    if isdir(_C_EXT_DEPENDENCY_DIR):
+        shutil.rmtree(_C_EXT_DEPENDENCY_DIR)
+    os.mkdir(_C_EXT_DEPENDENCY_DIR)
+    os.mkdir(_C_EXT_DEPENDENCY_LIB_LOCATION)
+    os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_DIR)
+    os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['ionc'])
+    os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['decNumber'])
+    # Move ion-c binaries to ion-c-build
+    _move_ionc()
 
 
 def _check_dependencies():
@@ -187,6 +186,7 @@ def _install_ionc():
     if not _library_exists():
         if not _download_ionc():
             return False
+    move_build_lib_for_distribution()
 
     return True
 

setup.py

@@ -17,25 +17,16 @@
 from __future__ import division
 from __future__ import print_function
 
+import sys
 from os.path import abspath, dirname, join, exists
-
 from setuptools import setup, find_packages, Extension
-
 from install import _install_ionc
 
-C_EXT = True
-c_ext_dir = abspath(join(dirname(__file__), 'amazon/ion/ion-c-build'))
-
-
-def c_ext_exist():
-    return exists(c_ext_dir) and exists(join(c_ext_dir, "lib")) and exists(join(c_ext_dir, "include"))
+C_EXT = True if not hasattr(sys, 'pypy_translation_info') else False
 
 
 def run_setup():
-    if C_EXT:
-        if not c_ext_exist():
-            print('Initialize C extension...')
-            _install_ionc()
+    if C_EXT and _install_ionc():
         print('C extension is enabled!')
         kw = dict(
             ext_modules=[
@@ -60,7 +51,7 @@ def run_setup():
 
     setup(
         name='amazon.ion',
-        version='0.7.98',
+        version='0.7.0',
         description='A Python implementation of Amazon Ion.',
         url='http://github.com/amzn/ion-python',
         author='Amazon Ion Team',

tests/test_simpleion.py

@@ -212,8 +212,12 @@ def generate_scalars_binary(scalars_map, preceding_symbols=0):
                 has_symbols = True
             elif ion_type is IonType.STRING:
                 # Encode all strings as symbols too.
-                symbol_expected = _serialize_symbol(
-                    IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10 + preceding_symbols)))
+                if c_ext:
+                    symbol_expected = _serialize_symbol(
+                        IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10)))
+                else:
+                    symbol_expected = _serialize_symbol(
+                        IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10 + preceding_symbols)))
                 yield _Parameter(IonType.SYMBOL.name + ' ' + native,
                                  IonPyText.from_value(IonType.SYMBOL, native), symbol_expected, True)
             yield _Parameter('%s %s' % (ion_type.name, native), native, native_expected, has_symbols)
@@ -260,22 +264,32 @@ def generate_annotated_values_binary(scalars_map, container_map):
         obj.ion_annotations = (_st(u'annot1'), _st(u'annot2'),)
         annot_length = 2  # 10 and 11 each fit in one VarUInt byte
         annot_length_length = 1  # 2 fits in one VarUInt byte
-        value_length = len(value_p.expected)
-        length_field = annot_length + annot_length_length + value_length
-        wrapper = []
-        _write_length(wrapper, length_field, 0xE0)
-        wrapper.extend([
-            VARUINT_END_BYTE | annot_length,
-            VARUINT_END_BYTE | 10,
-            VARUINT_END_BYTE | 11
-        ])
 
         final_expected = ()
         if isinstance(value_p.expected, (list, tuple)):
             expecteds = value_p.expected
         else:
             expecteds = (value_p.expected, )
         for one_expected in expecteds:
+            value_length = len(one_expected)
+            length_field = annot_length + annot_length_length + value_length
+            wrapper = []
+            _write_length(wrapper, length_field, 0xE0)
+
+            if c_ext and obj.ion_type is IonType.SYMBOL and not isinstance(obj, IonPyNull) \
+                    and not (hasattr(obj, 'sid') and (obj.sid < 10 or obj.sid is None)):
+                wrapper.extend([
+                    VARUINT_END_BYTE | annot_length,
+                    VARUINT_END_BYTE | 11,
+                    VARUINT_END_BYTE | 12
+                ])
+            else:
+                wrapper.extend([
+                    VARUINT_END_BYTE | annot_length,
+                    VARUINT_END_BYTE | 10,
+                    VARUINT_END_BYTE | 11
+                ])
+
             exp = bytearray(wrapper) + one_expected
             final_expected += (exp, )
 

tests/test_writer_binary_raw.py

@@ -185,19 +185,30 @@ def _generate_annotated_values():
             [SymbolToken(None, 10), SymbolToken(None, 11)]),) + value_p.events[1:]
         annot_length = 2  # 10 and 11 each fit in one VarUInt byte
         annot_length_length = 1  # 2 fits in one VarUInt byte
-        value_length = len(value_p.expected)
-        length_field = annot_length + annot_length_length + value_length
-        wrapper = []
-        _write_length(wrapper, length_field, 0xE0)
-        wrapper.extend([
-            VARUINT_END_BYTE | annot_length,
-            VARUINT_END_BYTE | 10,
-            VARUINT_END_BYTE | 11
-        ])
+        final_expected = ()
+        if isinstance(value_p.expected, (list, tuple)):
+            expecteds = value_p.expected
+        else:
+            expecteds = (value_p.expected,)
+
+        for one_expected in expecteds:
+            value_length = len(one_expected)
+            length_field = annot_length + annot_length_length + value_length
+            wrapper = []
+            _write_length(wrapper, length_field, 0xE0)
+            wrapper.extend([
+                VARUINT_END_BYTE | annot_length,
+                VARUINT_END_BYTE | 10,
+                VARUINT_END_BYTE | 11
+            ])
+
+            exp = bytearray(wrapper) + one_expected
+            final_expected += (exp, )
+
         yield _P(
             desc='ANN %s' % value_p.desc,
             events=events + (_E(_ET.STREAM_END),),
-            expected=bytearray(wrapper) + value_p.expected,
+            expected=final_expected,
         )