From b500aa1837bfb83d367e9fc0e2ca7e777e6b0bfa Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Mon, 9 Mar 2020 19:41:52 -0500 Subject: [PATCH 01/25] begin to implement list codec --- src/embed_tests/Codecs.cs | 27 ++++++++- src/runtime/Codecs/ListCodec.cs | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/runtime/Codecs/ListCodec.cs diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 18fcd32d1..76889b409 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -1,7 +1,7 @@ namespace Python.EmbeddingTest { using System; using System.Collections.Generic; - using System.Text; + using System.Linq; using NUnit.Framework; using Python.Runtime; using Python.Runtime.Codecs; @@ -82,6 +82,31 @@ static void TupleRoundtripGeneric() { Assert.AreEqual(expected: tuple, actual: restored); } } + + [Test] + public void ListCodecTest() + { + var codec = ListCodec.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var x = new PyList(items.ToArray()); + Assert.IsTrue(codec.CanDecode(x, typeof(List))); + Assert.IsTrue(codec.CanDecode(x, typeof(IList))); + Assert.IsTrue(codec.CanDecode(x, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(x, typeof(bool))); + + System.Collections.IEnumerable plainEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable); }); + Assert.IsNotNull(plainEnumerable); + IList list = null; + list = plainEnumerable.Cast().ToList(); + Assert.AreEqual(list.Count, 3); + Assert.AreEqual(list[0], 1); + Assert.AreEqual(list[1], 2); + Assert.AreEqual(list[2], 3); + } } /// diff --git a/src/runtime/Codecs/ListCodec.cs b/src/runtime/Codecs/ListCodec.cs new file mode 100644 index 000000000..2470241fe --- /dev/null +++ b/src/runtime/Codecs/ListCodec.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Runtime.Codecs +{ + class ListCodec : IPyObjectDecoder + { + public bool CanDecode(PyObject objectType, Type targetType) + { + //first check if the PyObject is iterable. + IntPtr IterObject = Runtime.PyObject_GetIter(objectType.Handle); + if (IterObject == IntPtr.Zero) + return false; + + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return true; + + //if its not a plain IEnumerable it must be a generic type + if (!targetType.IsGenericType) return false; + + Predicate IsCLRSequence = (Type type) => { + return (type.GetGenericTypeDefinition() == typeof(IEnumerable<>) || + type.GetGenericTypeDefinition() == typeof(ICollection<>) || + type.GetGenericTypeDefinition() == typeof(IList<>)); + }; + + if (IsCLRSequence(targetType)) + return true; + + //if it implements any of the standard C# collection interfaces, we can decode it. + foreach (Type itf in targetType.GetInterfaces()) + { + if (IsCLRSequence(itf)) + return true; + } + + //TODO objectType should implement the Sequence protocol to be convertible to ICollection + // and the list protocol to be convertible to IList. We should check for list first, + // then collection, then enumerable + + + //if we get here we cannot decode it. + return false; + } + + private class PyEnumerable : System.Collections.IEnumerable + { + PyObject iterObject; + internal PyEnumerable(PyObject pyObj) + { + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); + } + + public IEnumerator GetEnumerator() + { + IntPtr item; + while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + { + object obj = null; + if (!Converter.ToManaged(item, typeof(object), out obj, true)) + { + Runtime.XDecref(item); + break; + } + + Runtime.XDecref(item); + yield return obj; + } + } + } + + private object ToPlainEnumerable(PyObject pyObj) + { + return new PyEnumerable(pyObj); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + object var = null; + //first see if T is a plan IEnumerable + if (typeof(T) == typeof(System.Collections.IEnumerable)) + { + var = ToPlainEnumerable(pyObj); + } + + value = (T)var; + return false; + } + + public static ListCodec Instance { get; } = new ListCodec(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} From f80ae875359e32e530da508163be01cc70e7e2bf Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Mon, 9 Mar 2020 19:53:26 -0500 Subject: [PATCH 02/25] add a test --- src/embed_tests/Codecs.cs | 45 +++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 76889b409..67c10b2cf 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -97,15 +97,42 @@ public void ListCodecTest() Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); Assert.IsFalse(codec.CanDecode(x, typeof(bool))); - System.Collections.IEnumerable plainEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable); }); - Assert.IsNotNull(plainEnumerable); - IList list = null; - list = plainEnumerable.Cast().ToList(); - Assert.AreEqual(list.Count, 3); - Assert.AreEqual(list[0], 1); - Assert.AreEqual(list[1], 2); - Assert.AreEqual(list[2], 3); + Action checkPlainEnumerable = (System.Collections.IEnumerable enumerable) => + { + Assert.IsNotNull(enumerable); + IList list = null; + list = enumerable.Cast().ToList(); + Assert.AreEqual(list.Count, 3); + Assert.AreEqual(list[0], 1); + Assert.AreEqual(list[1], 2); + Assert.AreEqual(list[2], 3); + }; + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable1); }); + checkPlainEnumerable(plainEnumerable1); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var locals = new PyDict(); + PythonEngine.Exec(@" +class foo(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter +foo_instance = foo() +", null, locals.Handle); + + var foo = locals.GetItem("foo_instance"); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable2); }); + checkPlainEnumerable(plainEnumerable2); } } From 765b6a221bf6414d147e96c0310c8b1b0045d26d Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Tue, 10 Mar 2020 20:43:26 -0500 Subject: [PATCH 03/25] improve CanDecode --- src/embed_tests/Codecs.cs | 13 +++++ src/runtime/Codecs/ListCodec.cs | 94 +++++++++++++++++++++++++-------- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 67c10b2cf..4b5687c6d 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -133,6 +133,19 @@ raise StopIteration System.Collections.IEnumerable plainEnumerable2 = null; Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable2); }); checkPlainEnumerable(plainEnumerable2); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + + //cannot convert to ICollection or IList of any type since the python type is only iterable + Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); + + } } diff --git a/src/runtime/Codecs/ListCodec.cs b/src/runtime/Codecs/ListCodec.cs index 2470241fe..ff9c47940 100644 --- a/src/runtime/Codecs/ListCodec.cs +++ b/src/runtime/Codecs/ListCodec.cs @@ -8,43 +8,95 @@ namespace Python.Runtime.Codecs { class ListCodec : IPyObjectDecoder { - public bool CanDecode(PyObject objectType, Type targetType) + private enum CollectionRank + { + //order matters, this is in increasing order of specialization + None, + Iterable, + Sequence, + List + } + + private CollectionRank GetRank(PyObject objectType) { + var handle = objectType.Handle; //first check if the PyObject is iterable. - IntPtr IterObject = Runtime.PyObject_GetIter(objectType.Handle); + IntPtr IterObject = Runtime.PyObject_GetIter(handle); if (IterObject == IntPtr.Zero) - return false; + return CollectionRank.None; - //if it is a plain IEnumerable, we can decode it using sequence protocol. - if (targetType == typeof(System.Collections.IEnumerable)) - return true; + //now check if its a sequence + if (Runtime.PySequence_Check(handle)) + { + //last check if its a list + if (Runtime.PyList_Check(handle)) + return CollectionRank.List; + return CollectionRank.Sequence; + } - //if its not a plain IEnumerable it must be a generic type - if (!targetType.IsGenericType) return false; + return CollectionRank.Iterable; + } - Predicate IsCLRSequence = (Type type) => { - return (type.GetGenericTypeDefinition() == typeof(IEnumerable<>) || - type.GetGenericTypeDefinition() == typeof(ICollection<>) || - type.GetGenericTypeDefinition() == typeof(IList<>)); + private CollectionRank GetRank(Type targetType) + { + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return CollectionRank.Iterable; + + Func getRankOfType = (Type type) => { + if (type.GetGenericTypeDefinition() == typeof(IList<>)) + return CollectionRank.List; + if (type.GetGenericTypeDefinition() == typeof(ICollection<>)) + return CollectionRank.Sequence; + if (type.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + return CollectionRank.Iterable; + return CollectionRank.None; }; - if (IsCLRSequence(targetType)) - return true; + if (targetType.IsGenericType) + { + var thisRank = getRankOfType(targetType); + if (thisRank != CollectionRank.None) + return thisRank; + } + var maxRank = CollectionRank.None; //if it implements any of the standard C# collection interfaces, we can decode it. foreach (Type itf in targetType.GetInterfaces()) { - if (IsCLRSequence(itf)) - return true; + if (!itf.IsGenericType) continue; + + var thisRank = getRankOfType(itf); + + //this is the most specialized type. return early + if (thisRank == CollectionRank.List) return thisRank; + + //if it is more specialized, assign to max rank + if ((int)thisRank > (int)maxRank) + maxRank = thisRank; } - //TODO objectType should implement the Sequence protocol to be convertible to ICollection - // and the list protocol to be convertible to IList. We should check for list first, - // then collection, then enumerable + return maxRank; + } - //if we get here we cannot decode it. - return false; + public bool CanDecode(PyObject objectType, Type targetType) + { + //get the python object rank + var pyRank = GetRank(objectType); + if (pyRank == CollectionRank.None) + return false; + + //get the clr object rank + var clrRank = GetRank(targetType); + if (clrRank == CollectionRank.None) + return false; + + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return true; + + return (int)pyRank >= (int)clrRank; } private class PyEnumerable : System.Collections.IEnumerable From 98ce49ce66bc3dc987f87b1c9a786dfcf0fa3f77 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Tue, 10 Mar 2020 22:17:49 -0500 Subject: [PATCH 04/25] improve list codec --- src/embed_tests/Codecs.cs | 9 +++- src/runtime/Codecs/ListCodec.cs | 84 +++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 4b5687c6d..a94d381e0 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -90,13 +90,16 @@ public void ListCodecTest() var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; var x = new PyList(items.ToArray()); - Assert.IsTrue(codec.CanDecode(x, typeof(List))); Assert.IsTrue(codec.CanDecode(x, typeof(IList))); Assert.IsTrue(codec.CanDecode(x, typeof(System.Collections.IEnumerable))); Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); Assert.IsFalse(codec.CanDecode(x, typeof(bool))); + //we'd have to copy into a list to do this. not the best idea to support it. + //maybe there can be a flag on listcodec to allow it. + Assert.IsFalse(codec.CanDecode(x, typeof(List))); + Action checkPlainEnumerable = (System.Collections.IEnumerable enumerable) => { Assert.IsNotNull(enumerable); @@ -145,7 +148,9 @@ raise StopIteration Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); - + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intEnumerable); }); + checkPlainEnumerable(intEnumerable); } } diff --git a/src/runtime/Codecs/ListCodec.cs b/src/runtime/Codecs/ListCodec.cs index ff9c47940..a622f5c1c 100644 --- a/src/runtime/Codecs/ListCodec.cs +++ b/src/runtime/Codecs/ListCodec.cs @@ -37,11 +37,11 @@ private CollectionRank GetRank(PyObject objectType) return CollectionRank.Iterable; } - private CollectionRank GetRank(Type targetType) + private Tuple GetRankAndType(Type collectionType) { //if it is a plain IEnumerable, we can decode it using sequence protocol. - if (targetType == typeof(System.Collections.IEnumerable)) - return CollectionRank.Iterable; + if (collectionType == typeof(System.Collections.IEnumerable)) + return new Tuple(CollectionRank.Iterable, typeof(object)); Func getRankOfType = (Type type) => { if (type.GetGenericTypeDefinition() == typeof(IList<>)) @@ -53,32 +53,25 @@ private CollectionRank GetRank(Type targetType) return CollectionRank.None; }; - if (targetType.IsGenericType) - { - var thisRank = getRankOfType(targetType); - if (thisRank != CollectionRank.None) - return thisRank; - } - - var maxRank = CollectionRank.None; - //if it implements any of the standard C# collection interfaces, we can decode it. - foreach (Type itf in targetType.GetInterfaces()) + if (collectionType.IsGenericType) { - if (!itf.IsGenericType) continue; - - var thisRank = getRankOfType(itf); + //for compatibility we *could* do this and copy the value but probably not the best option. + /*if (collectionType.GetGenericTypeDefinition() == typeof(List<>)) + return new Tuple(CollectionRank.List, elementType);*/ - //this is the most specialized type. return early - if (thisRank == CollectionRank.List) return thisRank; - - //if it is more specialized, assign to max rank - if ((int)thisRank > (int)maxRank) - maxRank = thisRank; + var elementType = collectionType.GetGenericArguments()[0]; + var thisRank = getRankOfType(collectionType); + if (thisRank != CollectionRank.None) + return new Tuple(thisRank, elementType); } - return maxRank; + return null; } + private CollectionRank? GetRank(Type targetType) + { + return GetRankAndType(targetType)?.Item1; + } public bool CanDecode(PyObject objectType, Type targetType) { @@ -89,7 +82,7 @@ public bool CanDecode(PyObject objectType, Type targetType) //get the clr object rank var clrRank = GetRank(targetType); - if (clrRank == CollectionRank.None) + if (clrRank == null || clrRank == CollectionRank.None) return false; //if it is a plain IEnumerable, we can decode it using sequence protocol. @@ -99,15 +92,16 @@ public bool CanDecode(PyObject objectType, Type targetType) return (int)pyRank >= (int)clrRank; } - private class PyEnumerable : System.Collections.IEnumerable + private class GenericPyEnumerable : IEnumerable { - PyObject iterObject; - internal PyEnumerable(PyObject pyObj) + protected PyObject iterObject; + + internal GenericPyEnumerable(PyObject pyObj) { iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); } - public IEnumerator GetEnumerator() + IEnumerator IEnumerable.GetEnumerator() { IntPtr item; while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) @@ -123,11 +117,32 @@ public IEnumerator GetEnumerator() yield return obj; } } + + public IEnumerator GetEnumerator() + { + IntPtr item; + while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + { + object obj = null; + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Runtime.XDecref(item); + break; + } + + Runtime.XDecref(item); + yield return (T)obj; + } + } } private object ToPlainEnumerable(PyObject pyObj) { - return new PyEnumerable(pyObj); + return new GenericPyEnumerable(pyObj); + } + private object ToEnumerable(PyObject pyObj) + { + return new GenericPyEnumerable(pyObj); } public bool TryDecode(PyObject pyObj, out T value) @@ -136,7 +151,16 @@ public bool TryDecode(PyObject pyObj, out T value) //first see if T is a plan IEnumerable if (typeof(T) == typeof(System.Collections.IEnumerable)) { - var = ToPlainEnumerable(pyObj); + var = new GenericPyEnumerable(pyObj); + } + + //next use the rank to return the appropriate type + var clrRank = GetRank(typeof(T)); + if (clrRank == CollectionRank.Iterable) + var = new GenericPyEnumerable(pyObj); + else + { + //var = null; } value = (T)var; From b827f5a89a0320a6fe3dd417723697e860d382b8 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Tue, 10 Mar 2020 23:01:26 -0500 Subject: [PATCH 05/25] full set of list codecs --- src/embed_tests/Codecs.cs | 69 +++++++++-- src/runtime/Codecs/ListCodec.cs | 197 +++++++++++++++++++++++++++++--- 2 files changed, 243 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index a94d381e0..b71f882c9 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -100,7 +100,7 @@ public void ListCodecTest() //maybe there can be a flag on listcodec to allow it. Assert.IsFalse(codec.CanDecode(x, typeof(List))); - Action checkPlainEnumerable = (System.Collections.IEnumerable enumerable) => + Action checkUsingEnumerable = (System.Collections.IEnumerable enumerable) => { Assert.IsNotNull(enumerable); IList list = null; @@ -111,14 +111,69 @@ public void ListCodecTest() Assert.AreEqual(list[2], 3); }; + Action checkEmptyUsingEnumerable = (System.Collections.IEnumerable enumerable) => + { + Assert.IsNotNull(enumerable); + IList list = null; + list = enumerable.Cast().ToList(); + Assert.AreEqual(list.Count, 0); + }; + //ensure a PyList can be converted to a plain IEnumerable System.Collections.IEnumerable plainEnumerable1 = null; Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable1); }); - checkPlainEnumerable(plainEnumerable1); + checkUsingEnumerable(plainEnumerable1); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); + + //cannot convert to ICollection or IList of any type since the python type is only iterable + Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(x, typeof(IList))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intEnumerable); }); + checkUsingEnumerable(intEnumerable); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out doubleEnumerable); }); + checkUsingEnumerable(doubleEnumerable); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out stringEnumerable); }); + checkEmptyUsingEnumerable(stringEnumerable); + + Runtime.CheckExceptionOccurred(); + + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out stringCollection); }); + checkEmptyUsingEnumerable(stringCollection); + + Runtime.CheckExceptionOccurred(); + + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intCollection); }); + checkUsingEnumerable(intCollection); + + Runtime.CheckExceptionOccurred(); + + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intList); }); + checkUsingEnumerable(intList); //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable - var locals = new PyDict(); - PythonEngine.Exec(@" + var locals = new PyDict(); + using (Py.GIL()) + { + PythonEngine.Exec(@" class foo(): def __init__(self): self.counter = 0 @@ -131,11 +186,12 @@ raise StopIteration return self.counter foo_instance = foo() ", null, locals.Handle); + } var foo = locals.GetItem("foo_instance"); System.Collections.IEnumerable plainEnumerable2 = null; Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable2); }); - checkPlainEnumerable(plainEnumerable2); + checkUsingEnumerable(plainEnumerable2); //can convert to any generic ienumerable. If the type is not assignable from the python element //it will be an exception during TryDecode @@ -148,9 +204,8 @@ raise StopIteration Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); - IEnumerable intEnumerable = null; Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intEnumerable); }); - checkPlainEnumerable(intEnumerable); + checkUsingEnumerable(intEnumerable); } } diff --git a/src/runtime/Codecs/ListCodec.cs b/src/runtime/Codecs/ListCodec.cs index a622f5c1c..78c5f802f 100644 --- a/src/runtime/Codecs/ListCodec.cs +++ b/src/runtime/Codecs/ListCodec.cs @@ -75,6 +75,8 @@ private Tuple GetRankAndType(Type collectionType) public bool CanDecode(PyObject objectType, Type targetType) { + //TODO - convert pyTuple to IReadOnlyList + //get the python object rank var pyRank = GetRank(objectType); if (pyRank == CollectionRank.None) @@ -92,12 +94,14 @@ public bool CanDecode(PyObject objectType, Type targetType) return (int)pyRank >= (int)clrRank; } - private class GenericPyEnumerable : IEnumerable + private class PyEnumerable : IEnumerable { protected PyObject iterObject; + protected PyObject pyObject; - internal GenericPyEnumerable(PyObject pyObj) + public PyEnumerable(PyObject pyObj) { + pyObject = pyObj; iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); } @@ -109,6 +113,7 @@ IEnumerator IEnumerable.GetEnumerator() object obj = null; if (!Converter.ToManaged(item, typeof(object), out obj, true)) { + Exceptions.Clear(); Runtime.XDecref(item); break; } @@ -126,6 +131,7 @@ public IEnumerator GetEnumerator() object obj = null; if (!Converter.ToManaged(item, typeof(T), out obj, true)) { + Exceptions.Clear(); Runtime.XDecref(item); break; } @@ -136,35 +142,194 @@ public IEnumerator GetEnumerator() } } - private object ToPlainEnumerable(PyObject pyObj) + private class PyCollection : PyEnumerable, ICollection { - return new GenericPyEnumerable(pyObj); + public PyCollection(PyObject pyObj) : base(pyObj) + { + + } + + public int Count + { + get + { + return (int)Runtime.PySequence_Size(pyObject.Handle); + } + } + + public virtual bool IsReadOnly => false; + + public virtual void Add(T item) + { + //not implemented for Python sequence rank + throw new NotImplementedException(); + } + + public void Clear() + { + if (IsReadOnly) + throw new NotImplementedException(); + var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); + if (result == -1) + throw new Exception("failed to clear sequence"); + } + + public bool Contains(T item) + { + //not sure if IEquatable is implemented and this will work! + foreach (var element in this) + if (element.Equals(item)) return true; + + return false; + } + + protected T getItem(int index) + { + IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); + object obj; + + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Exceptions.Clear(); + Runtime.XDecref(item); + Exceptions.RaiseTypeError("wrong type in sequence"); + } + + return (T)obj; + } + + public void CopyTo(T[] array, int arrayIndex) + { + for (int index = 0; index < Count; index++) + { + array[index + arrayIndex] = getItem(index); + } + } + + protected bool removeAt(int index) + { + if (IsReadOnly) + throw new NotImplementedException(); + if (index >= Count || index < 0) + throw new IndexOutOfRangeException(); + + return Runtime.PySequence_DelItem(pyObject.Handle, index) != 0; + } + + protected int indexOf(T item) + { + var index = 0; + foreach (var element in this) + { + if (element.Equals(item)) return index; + index++; + } + + return -1; + } + + public bool Remove(T item) + { + return removeAt(indexOf(item)); + } } - private object ToEnumerable(PyObject pyObj) + + private class PyList : PyCollection, IList { - return new GenericPyEnumerable(pyObj); + public PyList(PyObject pyObj) : base(pyObj) + { + + } + + public T this[int index] + { + get + { + IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); + object obj; + + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Exceptions.Clear(); + Runtime.XDecref(item); + Exceptions.RaiseTypeError("wrong type in sequence"); + } + + return (T)obj; + } + set + { + IntPtr pyItem = Converter.ToPython(value, typeof(T)); + if (pyItem == IntPtr.Zero) + throw new Exception("failed to set item"); + + var result = Runtime.PySequence_SetItem(pyObject.Handle, index, pyItem); + Runtime.XDecref(pyItem); + if (result == -1) + throw new Exception("failed to set item"); + } + } + + public int IndexOf(T item) + { + return indexOf(item); + } + + public void Insert(int index, T item) + { + if (IsReadOnly) + throw new NotImplementedException(); + + IntPtr pyItem = Converter.ToPython(item, typeof(T)); + if (pyItem == IntPtr.Zero) + throw new Exception("failed to insert item"); + + var result = Runtime.PyList_Insert(pyObject.Handle, index, pyItem); + Runtime.XDecref(pyItem); + if (result == -1) + throw new Exception("failed to insert item"); + } + + public void RemoveAt(int index) + { + removeAt(index); + } } public bool TryDecode(PyObject pyObj, out T value) { - object var = null; //first see if T is a plan IEnumerable if (typeof(T) == typeof(System.Collections.IEnumerable)) { - var = new GenericPyEnumerable(pyObj); + object enumerable = new PyEnumerable(pyObj); + value = (T)enumerable; + return true; } //next use the rank to return the appropriate type - var clrRank = GetRank(typeof(T)); - if (clrRank == CollectionRank.Iterable) - var = new GenericPyEnumerable(pyObj); - else + var rankAndType = GetRankAndType(typeof(T)); + if (rankAndType.Item1 == CollectionRank.None) + throw new Exception("expected collection rank"); + + + var itemType = rankAndType.Item2; + Type collectionType = null; + if (rankAndType.Item1 == CollectionRank.Iterable) { - //var = null; + collectionType = typeof(PyEnumerable<>).MakeGenericType(itemType); } - - value = (T)var; - return false; + else if (rankAndType.Item1 == CollectionRank.Sequence) + { + collectionType = typeof(PyCollection<>).MakeGenericType(itemType); + } + else if (rankAndType.Item1 == CollectionRank.List) + { + collectionType = typeof(PyList<>).MakeGenericType(itemType); + } + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; } public static ListCodec Instance { get; } = new ListCodec(); From 224df0ea94e8c081052159106051797e487bef32 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 22 Mar 2020 09:45:42 -0500 Subject: [PATCH 06/25] abandon rank and use three codecs --- src/embed_tests/Codecs.cs | 579 ++++++++++-------- src/runtime/Codecs/IterableDecoder.cs | 57 ++ src/runtime/Codecs/ListCodec.cs | 342 ----------- src/runtime/Codecs/ListDecoder.cs | 49 ++ src/runtime/Codecs/SequenceDecoder.cs | 49 ++ .../CollectionWrappers/IterableWrapper.cs | 52 ++ src/runtime/CollectionWrappers/ListWrapper.cs | 66 ++ .../CollectionWrappers/SequenceWrapper.cs | 98 +++ 8 files changed, 705 insertions(+), 587 deletions(-) create mode 100644 src/runtime/Codecs/IterableDecoder.cs delete mode 100644 src/runtime/Codecs/ListCodec.cs create mode 100644 src/runtime/Codecs/ListDecoder.cs create mode 100644 src/runtime/Codecs/SequenceDecoder.cs create mode 100644 src/runtime/CollectionWrappers/IterableWrapper.cs create mode 100644 src/runtime/CollectionWrappers/ListWrapper.cs create mode 100644 src/runtime/CollectionWrappers/SequenceWrapper.cs diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index b71f882c9..cb27004ed 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -1,248 +1,337 @@ -namespace Python.EmbeddingTest { - using System; - using System.Collections.Generic; - using System.Linq; - using NUnit.Framework; - using Python.Runtime; - using Python.Runtime.Codecs; - - public class Codecs { - [SetUp] - public void SetUp() { - PythonEngine.Initialize(); - } - - [TearDown] - public void Dispose() { - PythonEngine.Shutdown(); - } - - [Test] - public void ConversionsGeneric() { - ConversionsGeneric, ValueTuple>(); - } - - static void ConversionsGeneric() { - TupleCodec.Register(); - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - T restored = default; - using (Py.GIL()) - using (var scope = Py.CreateScope()) { - void Accept(T value) => restored = value; - var accept = new Action(Accept).ToPython(); - scope.Set(nameof(tuple), tuple); - scope.Set(nameof(accept), accept); - scope.Exec($"{nameof(accept)}({nameof(tuple)})"); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void ConversionsObject() { - ConversionsObject, ValueTuple>(); - } - static void ConversionsObject() { - TupleCodec.Register(); - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - T restored = default; - using (Py.GIL()) - using (var scope = Py.CreateScope()) { - void Accept(object value) => restored = (T)value; - var accept = new Action(Accept).ToPython(); - scope.Set(nameof(tuple), tuple); - scope.Set(nameof(accept), accept); - scope.Exec($"{nameof(accept)}({nameof(tuple)})"); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void TupleRoundtripObject() { - TupleRoundtripObject, ValueTuple>(); - } - static void TupleRoundtripObject() { - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void TupleRoundtripGeneric() { - TupleRoundtripGeneric, ValueTuple>(); - } - - static void TupleRoundtripGeneric() { - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void ListCodecTest() - { - var codec = ListCodec.Instance; - var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; - - var x = new PyList(items.ToArray()); - Assert.IsTrue(codec.CanDecode(x, typeof(IList))); - Assert.IsTrue(codec.CanDecode(x, typeof(System.Collections.IEnumerable))); - Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(x, typeof(bool))); - - //we'd have to copy into a list to do this. not the best idea to support it. - //maybe there can be a flag on listcodec to allow it. - Assert.IsFalse(codec.CanDecode(x, typeof(List))); - - Action checkUsingEnumerable = (System.Collections.IEnumerable enumerable) => - { - Assert.IsNotNull(enumerable); - IList list = null; - list = enumerable.Cast().ToList(); - Assert.AreEqual(list.Count, 3); - Assert.AreEqual(list[0], 1); - Assert.AreEqual(list[1], 2); - Assert.AreEqual(list[2], 3); - }; - - Action checkEmptyUsingEnumerable = (System.Collections.IEnumerable enumerable) => - { - Assert.IsNotNull(enumerable); - IList list = null; - list = enumerable.Cast().ToList(); - Assert.AreEqual(list.Count, 0); - }; - - //ensure a PyList can be converted to a plain IEnumerable - System.Collections.IEnumerable plainEnumerable1 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable1); }); - checkUsingEnumerable(plainEnumerable1); - - //can convert to any generic ienumerable. If the type is not assignable from the python element - //it will be an exception during TryDecode - Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(x, typeof(IEnumerable))); - - //cannot convert to ICollection or IList of any type since the python type is only iterable - Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(x, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(x, typeof(IList))); - - IEnumerable intEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intEnumerable); }); - checkUsingEnumerable(intEnumerable); - - Runtime.CheckExceptionOccurred(); - - IEnumerable doubleEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out doubleEnumerable); }); - checkUsingEnumerable(doubleEnumerable); - - Runtime.CheckExceptionOccurred(); - - IEnumerable stringEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out stringEnumerable); }); - checkEmptyUsingEnumerable(stringEnumerable); - - Runtime.CheckExceptionOccurred(); - - ICollection stringCollection = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out stringCollection); }); - checkEmptyUsingEnumerable(stringCollection); - - Runtime.CheckExceptionOccurred(); - - ICollection intCollection = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intCollection); }); - checkUsingEnumerable(intCollection); - - Runtime.CheckExceptionOccurred(); - - IList intList = null; - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intList); }); - checkUsingEnumerable(intList); - - //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable - var locals = new PyDict(); - using (Py.GIL()) - { +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + static PyObject GetPythonIterable() + { + var locals = new PyDict(); + using (Py.GIL()) + { PythonEngine.Exec(@" -class foo(): - def __init__(self): - self.counter = 0 - def __iter__(self): +class foo(): + def __init__(self): + self.counter = 0 + def __iter__(self): return self - def __next__(self): - if self.counter == 3: - raise StopIteration - self.counter = self.counter + 1 - return self.counter + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter foo_instance = foo() -", null, locals.Handle); - } - - var foo = locals.GetItem("foo_instance"); - System.Collections.IEnumerable plainEnumerable2 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(x, out plainEnumerable2); }); - checkUsingEnumerable(plainEnumerable2); - - //can convert to any generic ienumerable. If the type is not assignable from the python element - //it will be an exception during TryDecode - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); - - //cannot convert to ICollection or IList of any type since the python type is only iterable - Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); - - Assert.DoesNotThrow(() => { codec.TryDecode>(x, out intEnumerable); }); - checkUsingEnumerable(intEnumerable); - } - } - - /// - /// "Decodes" only objects of exact type . - /// Result is just the raw proxy to the encoder instance itself. - /// - class ObjectToEncoderInstanceEncoder : IPyObjectEncoder - { - public bool CanEncode(Type type) => type == typeof(T); - public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); - } - - /// - /// Decodes object of specified Python type to the predefined value - /// - /// Type of the - class DecoderReturningPredefinedValue : IPyObjectDecoder - { - public PyObject TheOnlySupportedSourceType { get; } - public TTarget DecodeResult { get; } - - public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) - { - this.TheOnlySupportedSourceType = objectType; - this.DecodeResult = decodeResult; - } - - public bool CanDecode(PyObject objectType, Type targetType) - => objectType.Handle == TheOnlySupportedSourceType.Handle - && targetType == typeof(TTarget); - public bool TryDecode(PyObject pyObj, out T value) - { - if (typeof(T) != typeof(TTarget)) - throw new ArgumentException(nameof(T)); - value = (T)(object)DecodeResult; - return true; - } - } -} +", null, locals.Handle); + } + + return locals.GetItem("foo_instance"); + } + + [Test] + public void ListDecoderTest() + { + var codec = ListDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + Assert.IsTrue(codec.CanDecode(pyList, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + + //we'd have to copy into a list instance to do this, it would not be lossless. + //lossy converters can be implemented outside of the python.net core library + Assert.IsFalse(codec.CanDecode(pyList, typeof(List))); + + //convert to list of int + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); + CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); + + //convert to list of string. This will not work. + //The ListWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python list can be queried without any conversion, + //the IList will report a Count of 3. + IList stringList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); + Assert.AreEqual(stringList.Count, 3); + Assert.Throws(typeof(PythonException), ()=> { var x = stringList[0]; }); + + //can't convert python iterable to list (this will require a copy which isn't lossless) + var foo = GetPythonIterable(); + Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); + } + + [Test] + public void SequenceDecoderTest() + { + var codec = SequenceDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + //SequenceConverter can only convert to any ICollection + var pyList = new PyList(items.ToArray()); + //it can convert a PyList, since PyList satisfies the python sequence protocol + + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); + + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); + CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); + Assert.AreEqual(3, stringCollection.Count()); + Assert.Throws(typeof(PythonException), () => { + string[] array = new string[3]; + stringCollection.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + //can't convert python iterable to collection (this will require a copy which isn't lossless) + //python iterables do not satisfy the python sequence protocol + var foo = GetPythonIterable(); + Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); + + //python tuples do satisfy the python sequence protocol + var pyTuple = new PyObject(Runtime.PyTuple_New(3)); + + Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle); + Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle); + Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle); + + Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); + CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); + Assert.AreEqual(3, stringCollection2.Count()); + Assert.Throws(typeof(PythonException), () => { + string[] array = new string[3]; + stringCollection2.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } +} + + + [Test] + public void IterableDecoderTest() + { + var codec = IterableDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); + CollectionAssert.AreEqual(plainEnumerable1, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will lead to an empty iterable when decoding. TODO - should it throw? + Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); + CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); + + Assert.Throws(typeof(PythonException), () => { + foreach (string item in stringEnumerable) + { + var x = item; + } + }); + Assert.Throws(typeof(PythonException), () => { + stringEnumerable.Count(); + }); + + Runtime.CheckExceptionOccurred(); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var foo = GetPythonIterable(); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); + CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + } + } +} diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs new file mode 100644 index 000000000..747307248 --- /dev/null +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + class IterableDecoder : IPyObjectDecoder + { + internal static bool IsIterable(Type targetType) + { + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return true; + + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IEnumerable<>); + } + + internal static bool IsIterable(PyObject objectType) + { + //TODO - do I need to decref iterObject? + IntPtr iterObject = Runtime.PyObject_GetIter(objectType.Handle); + return iterObject != IntPtr.Zero; + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsIterable(objectType) && IsIterable(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + //first see if T is a plan IEnumerable + if (typeof(T) == typeof(System.Collections.IEnumerable)) + { + object enumerable = new CollectionWrappers.IterableWrapper(pyObj); + value = (T)enumerable; + return true; + } + + var elementType = typeof(T).GetGenericArguments()[0]; + var collectionType = typeof(CollectionWrappers.IterableWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static IterableDecoder Instance { get; } = new IterableDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/ListCodec.cs b/src/runtime/Codecs/ListCodec.cs deleted file mode 100644 index 78c5f802f..000000000 --- a/src/runtime/Codecs/ListCodec.cs +++ /dev/null @@ -1,342 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Python.Runtime.Codecs -{ - class ListCodec : IPyObjectDecoder - { - private enum CollectionRank - { - //order matters, this is in increasing order of specialization - None, - Iterable, - Sequence, - List - } - - private CollectionRank GetRank(PyObject objectType) - { - var handle = objectType.Handle; - //first check if the PyObject is iterable. - IntPtr IterObject = Runtime.PyObject_GetIter(handle); - if (IterObject == IntPtr.Zero) - return CollectionRank.None; - - //now check if its a sequence - if (Runtime.PySequence_Check(handle)) - { - //last check if its a list - if (Runtime.PyList_Check(handle)) - return CollectionRank.List; - return CollectionRank.Sequence; - } - - return CollectionRank.Iterable; - } - - private Tuple GetRankAndType(Type collectionType) - { - //if it is a plain IEnumerable, we can decode it using sequence protocol. - if (collectionType == typeof(System.Collections.IEnumerable)) - return new Tuple(CollectionRank.Iterable, typeof(object)); - - Func getRankOfType = (Type type) => { - if (type.GetGenericTypeDefinition() == typeof(IList<>)) - return CollectionRank.List; - if (type.GetGenericTypeDefinition() == typeof(ICollection<>)) - return CollectionRank.Sequence; - if (type.GetGenericTypeDefinition() == typeof(IEnumerable<>)) - return CollectionRank.Iterable; - return CollectionRank.None; - }; - - if (collectionType.IsGenericType) - { - //for compatibility we *could* do this and copy the value but probably not the best option. - /*if (collectionType.GetGenericTypeDefinition() == typeof(List<>)) - return new Tuple(CollectionRank.List, elementType);*/ - - var elementType = collectionType.GetGenericArguments()[0]; - var thisRank = getRankOfType(collectionType); - if (thisRank != CollectionRank.None) - return new Tuple(thisRank, elementType); - } - - return null; - } - - private CollectionRank? GetRank(Type targetType) - { - return GetRankAndType(targetType)?.Item1; - } - - public bool CanDecode(PyObject objectType, Type targetType) - { - //TODO - convert pyTuple to IReadOnlyList - - //get the python object rank - var pyRank = GetRank(objectType); - if (pyRank == CollectionRank.None) - return false; - - //get the clr object rank - var clrRank = GetRank(targetType); - if (clrRank == null || clrRank == CollectionRank.None) - return false; - - //if it is a plain IEnumerable, we can decode it using sequence protocol. - if (targetType == typeof(System.Collections.IEnumerable)) - return true; - - return (int)pyRank >= (int)clrRank; - } - - private class PyEnumerable : IEnumerable - { - protected PyObject iterObject; - protected PyObject pyObject; - - public PyEnumerable(PyObject pyObj) - { - pyObject = pyObj; - iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); - } - - IEnumerator IEnumerable.GetEnumerator() - { - IntPtr item; - while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) - { - object obj = null; - if (!Converter.ToManaged(item, typeof(object), out obj, true)) - { - Exceptions.Clear(); - Runtime.XDecref(item); - break; - } - - Runtime.XDecref(item); - yield return obj; - } - } - - public IEnumerator GetEnumerator() - { - IntPtr item; - while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) - { - object obj = null; - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Exceptions.Clear(); - Runtime.XDecref(item); - break; - } - - Runtime.XDecref(item); - yield return (T)obj; - } - } - } - - private class PyCollection : PyEnumerable, ICollection - { - public PyCollection(PyObject pyObj) : base(pyObj) - { - - } - - public int Count - { - get - { - return (int)Runtime.PySequence_Size(pyObject.Handle); - } - } - - public virtual bool IsReadOnly => false; - - public virtual void Add(T item) - { - //not implemented for Python sequence rank - throw new NotImplementedException(); - } - - public void Clear() - { - if (IsReadOnly) - throw new NotImplementedException(); - var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); - if (result == -1) - throw new Exception("failed to clear sequence"); - } - - public bool Contains(T item) - { - //not sure if IEquatable is implemented and this will work! - foreach (var element in this) - if (element.Equals(item)) return true; - - return false; - } - - protected T getItem(int index) - { - IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); - object obj; - - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Exceptions.Clear(); - Runtime.XDecref(item); - Exceptions.RaiseTypeError("wrong type in sequence"); - } - - return (T)obj; - } - - public void CopyTo(T[] array, int arrayIndex) - { - for (int index = 0; index < Count; index++) - { - array[index + arrayIndex] = getItem(index); - } - } - - protected bool removeAt(int index) - { - if (IsReadOnly) - throw new NotImplementedException(); - if (index >= Count || index < 0) - throw new IndexOutOfRangeException(); - - return Runtime.PySequence_DelItem(pyObject.Handle, index) != 0; - } - - protected int indexOf(T item) - { - var index = 0; - foreach (var element in this) - { - if (element.Equals(item)) return index; - index++; - } - - return -1; - } - - public bool Remove(T item) - { - return removeAt(indexOf(item)); - } - } - - private class PyList : PyCollection, IList - { - public PyList(PyObject pyObj) : base(pyObj) - { - - } - - public T this[int index] - { - get - { - IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); - object obj; - - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Exceptions.Clear(); - Runtime.XDecref(item); - Exceptions.RaiseTypeError("wrong type in sequence"); - } - - return (T)obj; - } - set - { - IntPtr pyItem = Converter.ToPython(value, typeof(T)); - if (pyItem == IntPtr.Zero) - throw new Exception("failed to set item"); - - var result = Runtime.PySequence_SetItem(pyObject.Handle, index, pyItem); - Runtime.XDecref(pyItem); - if (result == -1) - throw new Exception("failed to set item"); - } - } - - public int IndexOf(T item) - { - return indexOf(item); - } - - public void Insert(int index, T item) - { - if (IsReadOnly) - throw new NotImplementedException(); - - IntPtr pyItem = Converter.ToPython(item, typeof(T)); - if (pyItem == IntPtr.Zero) - throw new Exception("failed to insert item"); - - var result = Runtime.PyList_Insert(pyObject.Handle, index, pyItem); - Runtime.XDecref(pyItem); - if (result == -1) - throw new Exception("failed to insert item"); - } - - public void RemoveAt(int index) - { - removeAt(index); - } - } - - public bool TryDecode(PyObject pyObj, out T value) - { - //first see if T is a plan IEnumerable - if (typeof(T) == typeof(System.Collections.IEnumerable)) - { - object enumerable = new PyEnumerable(pyObj); - value = (T)enumerable; - return true; - } - - //next use the rank to return the appropriate type - var rankAndType = GetRankAndType(typeof(T)); - if (rankAndType.Item1 == CollectionRank.None) - throw new Exception("expected collection rank"); - - - var itemType = rankAndType.Item2; - Type collectionType = null; - if (rankAndType.Item1 == CollectionRank.Iterable) - { - collectionType = typeof(PyEnumerable<>).MakeGenericType(itemType); - } - else if (rankAndType.Item1 == CollectionRank.Sequence) - { - collectionType = typeof(PyCollection<>).MakeGenericType(itemType); - } - else if (rankAndType.Item1 == CollectionRank.List) - { - collectionType = typeof(PyList<>).MakeGenericType(itemType); - } - - var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); - value = (T)instance; - return true; - } - - public static ListCodec Instance { get; } = new ListCodec(); - - public static void Register() - { - PyObjectConversions.RegisterDecoder(Instance); - } - } -} diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs new file mode 100644 index 000000000..47a97a6c1 --- /dev/null +++ b/src/runtime/Codecs/ListDecoder.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + class ListDecoder : IPyObjectDecoder + { + private static bool IsList(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IList<>); + } + + private static bool IsList(PyObject objectType) + { + //must implement sequence protocol to fully implement list protocol + if (!SequenceDecoder.IsSequence(objectType)) return false; + + //returns wheter it implements the list protocol + return Runtime.PyList_Check(objectType.Handle); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsList(objectType) && IsList(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.ListWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static ListDecoder Instance { get; } = new ListDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs new file mode 100644 index 000000000..cbb15d0c7 --- /dev/null +++ b/src/runtime/Codecs/SequenceDecoder.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + class SequenceDecoder : IPyObjectDecoder + { + internal static bool IsSequence(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(ICollection<>); + } + + internal static bool IsSequence(PyObject objectType) + { + //must implement iterable protocol to fully implement sequence protocol + if (!IterableDecoder.IsIterable(objectType)) return false; + + //returns wheter it implements the sequence protocol + return Runtime.PySequence_Check(objectType.Handle); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsSequence(objectType) && IsSequence(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.SequenceWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static SequenceDecoder Instance { get; } = new SequenceDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs new file mode 100644 index 000000000..f0a8a1aa1 --- /dev/null +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Collections; + +namespace Python.Runtime.CollectionWrappers +{ + internal class IterableWrapper : IEnumerable + { + protected PyObject iterObject; + protected PyObject pyObject; + + public IterableWrapper(PyObject pyObj) + { + pyObject = pyObj; + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); + } + + IEnumerator IEnumerable.GetEnumerator() + { + IntPtr item; + while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + { + object obj = null; + if (!Converter.ToManaged(item, typeof(object), out obj, true)) + { + Runtime.XDecref(item); + Runtime.CheckExceptionOccurred(); + } + + Runtime.XDecref(item); + yield return obj; + } + } + + public IEnumerator GetEnumerator() + { + IntPtr item; + while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + { + object obj = null; + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Runtime.XDecref(item); + Runtime.CheckExceptionOccurred(); + } + + Runtime.XDecref(item); + yield return (T)obj; + } + } + } +} diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs new file mode 100644 index 000000000..0dcc43e83 --- /dev/null +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class ListWrapper : SequenceWrapper, IList + { + public ListWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public T this[int index] + { + get + { + IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); + object obj; + + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Runtime.XDecref(item); + Runtime.CheckExceptionOccurred(); + } + + return (T)obj; + } + set + { + IntPtr pyItem = Converter.ToPython(value, typeof(T)); + if (pyItem == IntPtr.Zero) + throw new Exception("failed to set item"); + + var result = Runtime.PySequence_SetItem(pyObject.Handle, index, pyItem); + Runtime.XDecref(pyItem); + if (result == -1) + throw new Exception("failed to set item"); + } + } + + public int IndexOf(T item) + { + return indexOf(item); + } + + public void Insert(int index, T item) + { + if (IsReadOnly) + throw new NotImplementedException(); + + IntPtr pyItem = Converter.ToPython(item, typeof(T)); + if (pyItem == IntPtr.Zero) + throw new Exception("failed to insert item"); + + var result = Runtime.PyList_Insert(pyObject.Handle, index, pyItem); + Runtime.XDecref(pyItem); + if (result == -1) + throw new Exception("failed to insert item"); + } + + public void RemoveAt(int index) + { + removeAt(index); + } + } +} diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs new file mode 100644 index 000000000..70ed18f75 --- /dev/null +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class SequenceWrapper : IterableWrapper, ICollection + { + public SequenceWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public int Count + { + get + { + return (int)Runtime.PySequence_Size(pyObject.Handle); + } + } + + public virtual bool IsReadOnly => false; + + public virtual void Add(T item) + { + //not implemented for Python sequence. + //ICollection is the closest analogue but it doesn't map perfectly. + //SequenceWrapper is not final and can be subclassed if necessary + throw new NotImplementedException(); + } + + public void Clear() + { + if (IsReadOnly) + throw new NotImplementedException(); + var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); + if (result == -1) + throw new Exception("failed to clear sequence"); + } + + public bool Contains(T item) + { + //not sure if IEquatable is implemented and this will work! + foreach (var element in this) + if (element.Equals(item)) return true; + + return false; + } + + protected T getItem(int index) + { + IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); + object obj; + + if (!Converter.ToManaged(item, typeof(T), out obj, true)) + { + Runtime.XDecref(item); + Runtime.CheckExceptionOccurred(); + } + + return (T)obj; + } + + public void CopyTo(T[] array, int arrayIndex) + { + for (int index = 0; index < Count; index++) + { + array[index + arrayIndex] = getItem(index); + } + } + + protected bool removeAt(int index) + { + if (IsReadOnly) + throw new NotImplementedException(); + if (index >= Count || index < 0) + throw new IndexOutOfRangeException(); + + return Runtime.PySequence_DelItem(pyObject.Handle, index) != 0; + } + + protected int indexOf(T item) + { + var index = 0; + foreach (var element in this) + { + if (element.Equals(item)) return index; + index++; + } + + return -1; + } + + public bool Remove(T item) + { + return removeAt(indexOf(item)); + } + } +} From 1600fdc9deb17ba5afc8c7c2e2bf495e0070c3e8 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 25 Mar 2020 17:45:21 -0500 Subject: [PATCH 07/25] update --- src/runtime/CollectionWrappers/ListWrapper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs index 0dcc43e83..b0a00e76e 100644 --- a/src/runtime/CollectionWrappers/ListWrapper.cs +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -52,7 +52,7 @@ public void Insert(int index, T item) if (pyItem == IntPtr.Zero) throw new Exception("failed to insert item"); - var result = Runtime.PyList_Insert(pyObject.Handle, index, pyItem); + var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem); Runtime.XDecref(pyItem); if (result == -1) throw new Exception("failed to insert item"); From c43446e7f339c9cd6eaf82549b13f1a9e15b0757 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 29 Mar 2020 13:04:00 -0500 Subject: [PATCH 08/25] respond to PR comments --- .../CollectionWrappers/IterableWrapper.cs | 20 +++++++++++++++++-- src/runtime/CollectionWrappers/ListWrapper.cs | 17 +++++++--------- .../CollectionWrappers/SequenceWrapper.cs | 14 +++++++++++-- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index f0a8a1aa1..f41fd2abd 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -6,18 +6,28 @@ namespace Python.Runtime.CollectionWrappers { internal class IterableWrapper : IEnumerable { - protected PyObject iterObject; protected PyObject pyObject; public IterableWrapper(PyObject pyObj) { pyObject = pyObj; - iterObject = new PyObject(Runtime.PyObject_GetIter(pyObj.Handle)); + } + + private void propagateIterationException() + { + var err = Runtime.PyErr_Occurred(); + if (err != null && err != Exceptions.StopIteration) + { + Runtime.CheckExceptionOccurred(); + } } IEnumerator IEnumerable.GetEnumerator() { + if (pyObject == null) yield break; + PyObject iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); IntPtr item; + while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) { object obj = null; @@ -30,10 +40,14 @@ IEnumerator IEnumerable.GetEnumerator() Runtime.XDecref(item); yield return obj; } + + propagateIterationException(); } public IEnumerator GetEnumerator() { + if (pyObject == null) yield break; + PyObject iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); IntPtr item; while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) { @@ -47,6 +61,8 @@ public IEnumerator GetEnumerator() Runtime.XDecref(item); yield return (T)obj; } + + propagateIterationException(); } } } diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs index b0a00e76e..0fa54b626 100644 --- a/src/runtime/CollectionWrappers/ListWrapper.cs +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -14,14 +14,11 @@ public T this[int index] { get { - IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); - object obj; + var item = Runtime.PyList_GetItem(pyObject.Handle, index); + var pyItem = new PyObject(item); - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Runtime.XDecref(item); + if (!Converter.ToManaged(pyItem.Handle, typeof(T), out object obj, true)) Runtime.CheckExceptionOccurred(); - } return (T)obj; } @@ -31,10 +28,10 @@ public T this[int index] if (pyItem == IntPtr.Zero) throw new Exception("failed to set item"); - var result = Runtime.PySequence_SetItem(pyObject.Handle, index, pyItem); + var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem); Runtime.XDecref(pyItem); if (result == -1) - throw new Exception("failed to set item"); + Runtime.CheckExceptionOccurred(); } } @@ -46,7 +43,7 @@ public int IndexOf(T item) public void Insert(int index, T item) { if (IsReadOnly) - throw new NotImplementedException(); + throw new InvalidOperationException("Collection is read-only"); IntPtr pyItem = Converter.ToPython(item, typeof(T)); if (pyItem == IntPtr.Zero) @@ -55,7 +52,7 @@ public void Insert(int index, T item) var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem); Runtime.XDecref(pyItem); if (result == -1) - throw new Exception("failed to insert item"); + Runtime.CheckExceptionOccurred(); } public void RemoveAt(int index) diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index 70ed18f75..da90f3154 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -14,7 +14,14 @@ public int Count { get { - return (int)Runtime.PySequence_Size(pyObject.Handle); + var size = Runtime.PySequence_Size(pyObject.Handle); + if (size == -1) + { + Runtime.CheckExceptionOccurred(); + throw new Exception("Unable to get sequence size!"); + } + + return (int)size; } } @@ -34,7 +41,10 @@ public void Clear() throw new NotImplementedException(); var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); if (result == -1) + { + Runtime.CheckExceptionOccurred(); throw new Exception("failed to clear sequence"); + } } public bool Contains(T item) @@ -46,7 +56,7 @@ public bool Contains(T item) return false; } - protected T getItem(int index) + private T getItem(int index) { IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); object obj; From 32fa32d5878a1937a5c5f9e3149abf8a7a84f949 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Tue, 31 Mar 2020 16:45:55 -0500 Subject: [PATCH 09/25] make decoders public --- src/runtime/Codecs/IterableDecoder.cs | 2 +- src/runtime/Codecs/ListDecoder.cs | 2 +- src/runtime/Codecs/SequenceDecoder.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs index 747307248..d66cade1b 100644 --- a/src/runtime/Codecs/IterableDecoder.cs +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -3,7 +3,7 @@ namespace Python.Runtime.Codecs { - class IterableDecoder : IPyObjectDecoder + public class IterableDecoder : IPyObjectDecoder { internal static bool IsIterable(Type targetType) { diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs index 47a97a6c1..69f0e61ab 100644 --- a/src/runtime/Codecs/ListDecoder.cs +++ b/src/runtime/Codecs/ListDecoder.cs @@ -3,7 +3,7 @@ namespace Python.Runtime.Codecs { - class ListDecoder : IPyObjectDecoder + public class ListDecoder : IPyObjectDecoder { private static bool IsList(Type targetType) { diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs index cbb15d0c7..88680f355 100644 --- a/src/runtime/Codecs/SequenceDecoder.cs +++ b/src/runtime/Codecs/SequenceDecoder.cs @@ -3,7 +3,7 @@ namespace Python.Runtime.Codecs { - class SequenceDecoder : IPyObjectDecoder + public class SequenceDecoder : IPyObjectDecoder { internal static bool IsSequence(Type targetType) { From dfc814b1545758f504682529ffcdb3256d036d24 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 1 Apr 2020 14:39:28 -0500 Subject: [PATCH 10/25] Make reset public --- src/runtime/converterextensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index b10d0c59f..3a9f18105 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -154,7 +154,7 @@ bool TryDecode(IntPtr pyHandle, out object result) #endregion - internal static void Reset() + public static void Reset() { lock (encoders) lock (decoders) From f025cd1e8e6a6bbf1a22fcadee40d219dc0c32f8 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 1 Apr 2020 14:40:08 -0500 Subject: [PATCH 11/25] add codec tests --- src/embed_tests/Codecs.cs | 205 ++++++++++-------- src/runtime/Codecs/IterableDecoder.cs | 4 +- src/runtime/Codecs/ListDecoder.cs | 4 +- src/runtime/Codecs/SequenceDecoder.cs | 7 +- .../CollectionWrappers/IterableWrapper.cs | 17 +- src/testing/CodecTest.cs | 47 ++++ src/testing/Python.Test.csproj | 1 - src/tests/test_codec.py | 70 ++++++ tests/tests.pyproj | 1 + 9 files changed, 256 insertions(+), 100 deletions(-) create mode 100644 src/testing/CodecTest.cs create mode 100644 src/tests/test_codec.py diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index cb27004ed..5cae73481 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -6,28 +6,34 @@ namespace Python.EmbeddingTest { using Python.Runtime; using Python.Runtime.Codecs; - public class Codecs { + public class Codecs + { [SetUp] - public void SetUp() { + public void SetUp() + { PythonEngine.Initialize(); } [TearDown] - public void Dispose() { + public void Dispose() + { PythonEngine.Shutdown(); } [Test] - public void ConversionsGeneric() { - ConversionsGeneric, ValueTuple>(); + public void TupleConversionsGeneric() + { + TupleConversionsGeneric, ValueTuple>(); } - static void ConversionsGeneric() { + static void TupleConversionsGeneric() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(T value) => restored = value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -38,15 +44,18 @@ static void ConversionsGeneric() { } [Test] - public void ConversionsObject() { - ConversionsObject, ValueTuple>(); + public void TupleConversionsObject() + { + TupleConversionsObject, ValueTuple>(); } - static void ConversionsObject() { + static void TupleConversionsObject() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(object value) => restored = (T)value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -57,12 +66,15 @@ static void ConversionsObject() { } [Test] - public void TupleRoundtripObject() { + public void TupleRoundtripObject() + { TupleRoundtripObject, ValueTuple>(); } - static void TupleRoundtripObject() { + static void TupleRoundtripObject() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); Assert.AreEqual(expected: tuple, actual: restored); @@ -70,25 +82,26 @@ static void TupleRoundtripObject() { } [Test] - public void TupleRoundtripGeneric() { + public void TupleRoundtripGeneric() + { TupleRoundtripGeneric, ValueTuple>(); } - static void TupleRoundtripGeneric() { + static void TupleRoundtripGeneric() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); Assert.AreEqual(expected: tuple, actual: restored); } } - static PyObject GetPythonIterable() + static string GetIntIterableCommands(string instanceName) { - var locals = new PyDict(); - using (Py.GIL()) - { - PythonEngine.Exec(@" + var builder = new System.Text.StringBuilder(); + builder.AppendLine(@" class foo(): def __init__(self): self.counter = 0 @@ -98,9 +111,18 @@ def __next__(self): if self.counter == 3: raise StopIteration self.counter = self.counter + 1 - return self.counter -foo_instance = foo() -", null, locals.Handle); + return self.counter"); + + builder.AppendLine(instanceName + " = foo()"); + return builder.ToString(); + } + + static PyObject GetPythonIterable() + { + var locals = new PyDict(); + using (Py.GIL()) + { + PythonEngine.Exec(GetIntIterableCommands("foo_instance"), null, locals.Handle); } return locals.GetItem("foo_instance"); @@ -113,16 +135,18 @@ public void ListDecoderTest() var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; var pyList = new PyList(items.ToArray()); - Assert.IsTrue(codec.CanDecode(pyList, typeof(IList))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(IList))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + + var pyListType = pyList.GetPythonType(); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); //we'd have to copy into a list instance to do this, it would not be lossless. //lossy converters can be implemented outside of the python.net core library - Assert.IsFalse(codec.CanDecode(pyList, typeof(List))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); //convert to list of int IList intList = null; @@ -137,11 +161,12 @@ public void ListDecoderTest() IList stringList = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); Assert.AreEqual(stringList.Count, 3); - Assert.Throws(typeof(PythonException), ()=> { var x = stringList[0]; }); + Assert.Throws(typeof(PythonException), () => { var x = stringList[0]; }); //can't convert python iterable to list (this will require a copy which isn't lossless) var foo = GetPythonIterable(); - Assert.IsFalse(codec.CanDecode(foo, typeof(IList))); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); } [Test] @@ -189,18 +214,20 @@ public void SequenceDecoderTest() //can't convert python iterable to collection (this will require a copy which isn't lossless) //python iterables do not satisfy the python sequence protocol var foo = GetPythonIterable(); - Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection))); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); //python tuples do satisfy the python sequence protocol var pyTuple = new PyObject(Runtime.PyTuple_New(3)); + var pyTupleType = pyTuple.GetPythonType(); Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle); Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle); Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle); - Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); //convert to collection of int ICollection intCollection2 = null; @@ -226,46 +253,6 @@ public void SequenceDecoderTest() Runtime.CheckExceptionOccurred(); } - } - - /// - /// "Decodes" only objects of exact type . - /// Result is just the raw proxy to the encoder instance itself. - /// - class ObjectToEncoderInstanceEncoder : IPyObjectEncoder - { - public bool CanEncode(Type type) => type == typeof(T); - public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); - } - - /// - /// Decodes object of specified Python type to the predefined value - /// - /// Type of the - class DecoderReturningPredefinedValue : IPyObjectDecoder - { - public PyObject TheOnlySupportedSourceType { get; } - public TTarget DecodeResult { get; } - - public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) - { - this.TheOnlySupportedSourceType = objectType; - this.DecodeResult = decodeResult; - } - - public bool CanDecode(PyObject objectType, Type targetType) - => objectType.Handle == TheOnlySupportedSourceType.Handle - && targetType == typeof(TTarget); - public bool TryDecode(PyObject pyObj, out T value) - { - if (typeof(T) != typeof(TTarget)) - throw new ArgumentException(nameof(T)); - value = (T)(object)DecodeResult; - return true; - } - } -} - [Test] public void IterableDecoderTest() @@ -274,11 +261,12 @@ public void IterableDecoderTest() var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; var pyList = new PyList(items.ToArray()); - Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + var pyListType = pyList.GetPythonType(); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); //ensure a PyList can be converted to a plain IEnumerable System.Collections.IEnumerable plainEnumerable1 = null; @@ -287,9 +275,9 @@ public void IterableDecoderTest() //can convert to any generic ienumerable. If the type is not assignable from the python element //it will lead to an empty iterable when decoding. TODO - should it throw? - Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); IEnumerable intEnumerable = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); @@ -320,18 +308,57 @@ public void IterableDecoderTest() //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); System.Collections.IEnumerable plainEnumerable2 = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); //can convert to any generic ienumerable. If the type is not assignable from the python element //it will be an exception during TryDecode - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); } } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } } diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs index d66cade1b..346057238 100644 --- a/src/runtime/Codecs/IterableDecoder.cs +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -19,9 +19,7 @@ internal static bool IsIterable(Type targetType) internal static bool IsIterable(PyObject objectType) { - //TODO - do I need to decref iterObject? - IntPtr iterObject = Runtime.PyObject_GetIter(objectType.Handle); - return iterObject != IntPtr.Zero; + return objectType.HasAttr("__iter__"); } public bool CanDecode(PyObject objectType, Type targetType) diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs index 69f0e61ab..23a87f2cc 100644 --- a/src/runtime/Codecs/ListDecoder.cs +++ b/src/runtime/Codecs/ListDecoder.cs @@ -18,8 +18,8 @@ private static bool IsList(PyObject objectType) //must implement sequence protocol to fully implement list protocol if (!SequenceDecoder.IsSequence(objectType)) return false; - //returns wheter it implements the list protocol - return Runtime.PyList_Check(objectType.Handle); + //returns wheter the type is a list. + return objectType.Handle == Runtime.PyListType; } public bool CanDecode(PyObject objectType, Type targetType) diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs index 88680f355..dce08fd99 100644 --- a/src/runtime/Codecs/SequenceDecoder.cs +++ b/src/runtime/Codecs/SequenceDecoder.cs @@ -18,8 +18,11 @@ internal static bool IsSequence(PyObject objectType) //must implement iterable protocol to fully implement sequence protocol if (!IterableDecoder.IsIterable(objectType)) return false; - //returns wheter it implements the sequence protocol - return Runtime.PySequence_Check(objectType.Handle); + //returns wheter it implements the sequence protocol + //according to python doc this needs to exclude dict subclasses + //but I don't know how to look for that given the objectType + //rather than the instance. + return objectType.HasAttr("__getitem__"); } public bool CanDecode(PyObject objectType, Type targetType) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index f41fd2abd..04b549517 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -47,10 +47,21 @@ IEnumerator IEnumerable.GetEnumerator() public IEnumerator GetEnumerator() { if (pyObject == null) yield break; - PyObject iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); - IntPtr item; - while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + PyObject iterObject = null; + using (Py.GIL()) { + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); + } + + while (true) + { + IntPtr item = IntPtr.Zero; + using (Py.GIL()) + { + item = Runtime.PyIter_Next(iterObject.Handle); + } + if (item == IntPtr.Zero) break; + object obj = null; if (!Converter.ToManaged(item, typeof(T), out obj, true)) { diff --git a/src/testing/CodecTest.cs b/src/testing/CodecTest.cs new file mode 100644 index 000000000..74f77531b --- /dev/null +++ b/src/testing/CodecTest.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Test +{ + public class ListMember + { + public ListMember(int value, string name) + { + Value = value; + Name = name; + } + + public int Value { get; set; } + public string Name { get; set; } + } + + public class ListConversionTester + { + public int GetLength(IEnumerable o) + { + return o.Count(); + } + public int GetLength(ICollection o) + { + return o.Count; + } + public int GetLength(IList o) + { + return o.Count; + } + public int GetLength2(IEnumerable o) + { + return o.Count(); + } + public int GetLength2(ICollection o) + { + return o.Count; + } + public int GetLength2(IList o) + { + return o.Count; + } + } +} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 4b7e4d93b..f7bc10bb4 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -4,7 +4,6 @@ true true - diff --git a/src/tests/test_codec.py b/src/tests/test_codec.py new file mode 100644 index 000000000..9fdbfbbf5 --- /dev/null +++ b/src/tests/test_codec.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +"""Test conversions using codecs from client python code""" +import clr +import System +import pytest +import Python.Runtime +from Python.Test import ListConversionTester, ListMember + +class int_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter + +class obj_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return ListMember(self.counter, "Number " + str(self.counter)) + +def test_iterable(): + """Test that a python iterable can be passed into a function that takes an IEnumerable""" + + #Python.Runtime.Codecs.ListDecoder.Register() + #Python.Runtime.Codecs.SequenceDecoder.Register() + Python.Runtime.Codecs.IterableDecoder.Register() + ob = ListConversionTester() + + iterable = int_iterable() + assert 3 == ob.GetLength(iterable) + + iterable2 = obj_iterable() + assert 3 == ob.GetLength2(iterable2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_sequence(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + tup = (1,2,3) + assert 3 == ob.GetLength(tup) + + tup2 = (ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")) + assert 3 == ob.GetLength(tup2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_list(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + l = [1,2,3] + assert 3 == ob.GetLength(l) + + l2 = [ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")] + assert 3 == ob.GetLength(l2) + + Python.Runtime.PyObjectConversions.Reset() diff --git a/tests/tests.pyproj b/tests/tests.pyproj index 439bc856a..fc1053f7b 100644 --- a/tests/tests.pyproj +++ b/tests/tests.pyproj @@ -52,6 +52,7 @@ + From 24d78c35e49787c41b1743cc75b2237b648f8de7 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 5 Apr 2020 21:34:42 -0500 Subject: [PATCH 12/25] respond to comments --- .../CollectionWrappers/IterableWrapper.cs | 32 ++++++++++++++----- src/runtime/CollectionWrappers/ListWrapper.cs | 16 +++++++--- .../CollectionWrappers/SequenceWrapper.cs | 15 +++++++-- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index 04b549517..554605025 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -6,30 +6,47 @@ namespace Python.Runtime.CollectionWrappers { internal class IterableWrapper : IEnumerable { - protected PyObject pyObject; + protected readonly PyObject pyObject; public IterableWrapper(PyObject pyObj) { + if (pyObj == null) + throw new PythonException(); pyObject = pyObj; } private void propagateIterationException() { var err = Runtime.PyErr_Occurred(); - if (err != null && err != Exceptions.StopIteration) + if (err == null) return; + + //remove StopIteration exceptions + if (0 != Runtime.PyErr_ExceptionMatches(Exceptions.StopIteration)) { - Runtime.CheckExceptionOccurred(); + Runtime.PyErr_Clear(); + return; } + + Runtime.CheckExceptionOccurred(); } IEnumerator IEnumerable.GetEnumerator() { - if (pyObject == null) yield break; - PyObject iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); - IntPtr item; + PyObject iterObject = null; + using (Py.GIL()) + { + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); + } - while ((item = Runtime.PyIter_Next(iterObject.Handle)) != IntPtr.Zero) + while (true) { + IntPtr item = IntPtr.Zero; + using (Py.GIL()) + { + item = Runtime.PyIter_Next(iterObject.Handle); + } + if (item == IntPtr.Zero) break; + object obj = null; if (!Converter.ToManaged(item, typeof(object), out obj, true)) { @@ -46,7 +63,6 @@ IEnumerator IEnumerable.GetEnumerator() public IEnumerator GetEnumerator() { - if (pyObject == null) yield break; PyObject iterObject = null; using (Py.GIL()) { diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs index 0fa54b626..6336151e6 100644 --- a/src/runtime/CollectionWrappers/ListWrapper.cs +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -26,10 +26,13 @@ public T this[int index] { IntPtr pyItem = Converter.ToPython(value, typeof(T)); if (pyItem == IntPtr.Zero) - throw new Exception("failed to set item"); + { + throw new InvalidCastException( + "cannot cast " + value.ToString() + "to type: " + typeof(T).ToString(), + new PythonException()); + } var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem); - Runtime.XDecref(pyItem); if (result == -1) Runtime.CheckExceptionOccurred(); } @@ -47,7 +50,7 @@ public void Insert(int index, T item) IntPtr pyItem = Converter.ToPython(item, typeof(T)); if (pyItem == IntPtr.Zero) - throw new Exception("failed to insert item"); + throw new PythonException(); var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem); Runtime.XDecref(pyItem); @@ -57,7 +60,12 @@ public void Insert(int index, T item) public void RemoveAt(int index) { - removeAt(index); + var result = removeAt(index); + + //PySequence_DelItem will set an error if it fails. throw it here + //since RemoveAt does not have a bool return value. + if (result == false) + Runtime.CheckExceptionOccurred(); } } } diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index da90f3154..cd6f9d8fb 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -72,9 +72,11 @@ private T getItem(int index) public void CopyTo(T[] array, int arrayIndex) { - for (int index = 0; index < Count; index++) + var index = 0; + foreach (var item in this) { - array[index + arrayIndex] = getItem(index); + array[index + arrayIndex] = item; + index++; } } @@ -102,7 +104,14 @@ protected int indexOf(T item) public bool Remove(T item) { - return removeAt(indexOf(item)); + var result = removeAt(indexOf(item)); + + //clear the python exception from PySequence_DelItem + //it is idiomatic in C# to return a bool rather than + //throw for a failed Remove in ICollection + if (result == false) + Runtime.PyErr_Clear(); + return result; } } } From ebcfa30da90121bda4c269fe63f956879cbbc538 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 30 Aug 2020 17:19:17 -0500 Subject: [PATCH 13/25] fix brace --- src/embed_tests/Codecs.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 5cae73481..126981618 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -323,7 +323,6 @@ public void IterableDecoderTest() CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); } } - } /// /// "Decodes" only objects of exact type . From 5cc575d73d3031dfec3b2864ef477f53eb6c8da3 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 30 Aug 2020 17:20:36 -0500 Subject: [PATCH 14/25] use unix line endings to avoid large diff --- src/embed_tests/Codecs.cs | 726 +++++++++++++++++++------------------- 1 file changed, 363 insertions(+), 363 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 126981618..06bd5ac72 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -1,363 +1,363 @@ -namespace Python.EmbeddingTest { - using System; - using System.Collections.Generic; - using System.Linq; - using NUnit.Framework; - using Python.Runtime; - using Python.Runtime.Codecs; - - public class Codecs - { - [SetUp] - public void SetUp() - { - PythonEngine.Initialize(); - } - - [TearDown] - public void Dispose() - { - PythonEngine.Shutdown(); - } - - [Test] - public void TupleConversionsGeneric() - { - TupleConversionsGeneric, ValueTuple>(); - } - - static void TupleConversionsGeneric() - { - TupleCodec.Register(); - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - T restored = default; - using (Py.GIL()) - using (var scope = Py.CreateScope()) - { - void Accept(T value) => restored = value; - var accept = new Action(Accept).ToPython(); - scope.Set(nameof(tuple), tuple); - scope.Set(nameof(accept), accept); - scope.Exec($"{nameof(accept)}({nameof(tuple)})"); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void TupleConversionsObject() - { - TupleConversionsObject, ValueTuple>(); - } - static void TupleConversionsObject() - { - TupleCodec.Register(); - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - T restored = default; - using (Py.GIL()) - using (var scope = Py.CreateScope()) - { - void Accept(object value) => restored = (T)value; - var accept = new Action(Accept).ToPython(); - scope.Set(nameof(tuple), tuple); - scope.Set(nameof(accept), accept); - scope.Exec($"{nameof(accept)}({nameof(tuple)})"); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void TupleRoundtripObject() - { - TupleRoundtripObject, ValueTuple>(); - } - static void TupleRoundtripObject() - { - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - [Test] - public void TupleRoundtripGeneric() - { - TupleRoundtripGeneric, ValueTuple>(); - } - - static void TupleRoundtripGeneric() - { - var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } - } - - static string GetIntIterableCommands(string instanceName) - { - var builder = new System.Text.StringBuilder(); - builder.AppendLine(@" -class foo(): - def __init__(self): - self.counter = 0 - def __iter__(self): - return self - def __next__(self): - if self.counter == 3: - raise StopIteration - self.counter = self.counter + 1 - return self.counter"); - - builder.AppendLine(instanceName + " = foo()"); - return builder.ToString(); - } - - static PyObject GetPythonIterable() - { - var locals = new PyDict(); - using (Py.GIL()) - { - PythonEngine.Exec(GetIntIterableCommands("foo_instance"), null, locals.Handle); - } - - return locals.GetItem("foo_instance"); - } - - [Test] - public void ListDecoderTest() - { - var codec = ListDecoder.Instance; - var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; - - var pyList = new PyList(items.ToArray()); - - var pyListType = pyList.GetPythonType(); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); - - //we'd have to copy into a list instance to do this, it would not be lossless. - //lossy converters can be implemented outside of the python.net core library - Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); - - //convert to list of int - IList intList = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); - CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); - - //convert to list of string. This will not work. - //The ListWrapper class will throw a python exception when it tries to access any element. - //TryDecode is a lossless conversion so there will be no exception at that point - //interestingly, since the size of the python list can be queried without any conversion, - //the IList will report a Count of 3. - IList stringList = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); - Assert.AreEqual(stringList.Count, 3); - Assert.Throws(typeof(PythonException), () => { var x = stringList[0]; }); - - //can't convert python iterable to list (this will require a copy which isn't lossless) - var foo = GetPythonIterable(); - var fooType = foo.GetPythonType(); - Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); - } - - [Test] - public void SequenceDecoderTest() - { - var codec = SequenceDecoder.Instance; - var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; - - //SequenceConverter can only convert to any ICollection - var pyList = new PyList(items.ToArray()); - //it can convert a PyList, since PyList satisfies the python sequence protocol - - Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); - - Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); - - //convert to collection of int - ICollection intCollection = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); - CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); - - //no python exception should have occurred during the above conversion and check - Runtime.CheckExceptionOccurred(); - - //convert to collection of string. This will not work. - //The SequenceWrapper class will throw a python exception when it tries to access any element. - //TryDecode is a lossless conversion so there will be no exception at that point - //interestingly, since the size of the python sequence can be queried without any conversion, - //the IList will report a Count of 3. - ICollection stringCollection = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); - Assert.AreEqual(3, stringCollection.Count()); - Assert.Throws(typeof(PythonException), () => { - string[] array = new string[3]; - stringCollection.CopyTo(array, 0); - }); - - Runtime.CheckExceptionOccurred(); - - //can't convert python iterable to collection (this will require a copy which isn't lossless) - //python iterables do not satisfy the python sequence protocol - var foo = GetPythonIterable(); - var fooType = foo.GetPythonType(); - Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); - - //python tuples do satisfy the python sequence protocol - var pyTuple = new PyObject(Runtime.PyTuple_New(3)); - var pyTupleType = pyTuple.GetPythonType(); - - Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle); - Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle); - Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle); - - Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); - Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); - - //convert to collection of int - ICollection intCollection2 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); - CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); - - //no python exception should have occurred during the above conversion and check - Runtime.CheckExceptionOccurred(); - - //convert to collection of string. This will not work. - //The SequenceWrapper class will throw a python exception when it tries to access any element. - //TryDecode is a lossless conversion so there will be no exception at that point - //interestingly, since the size of the python sequence can be queried without any conversion, - //the IList will report a Count of 3. - ICollection stringCollection2 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); - Assert.AreEqual(3, stringCollection2.Count()); - Assert.Throws(typeof(PythonException), () => { - string[] array = new string[3]; - stringCollection2.CopyTo(array, 0); - }); - - Runtime.CheckExceptionOccurred(); - - } - - [Test] - public void IterableDecoderTest() - { - var codec = IterableDecoder.Instance; - var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; - - var pyList = new PyList(items.ToArray()); - var pyListType = pyList.GetPythonType(); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); - Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); - - //ensure a PyList can be converted to a plain IEnumerable - System.Collections.IEnumerable plainEnumerable1 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); - CollectionAssert.AreEqual(plainEnumerable1, new List { 1, 2, 3 }); - - //can convert to any generic ienumerable. If the type is not assignable from the python element - //it will lead to an empty iterable when decoding. TODO - should it throw? - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); - - IEnumerable intEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); - CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); - - Runtime.CheckExceptionOccurred(); - - IEnumerable doubleEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); - CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); - - Runtime.CheckExceptionOccurred(); - - IEnumerable stringEnumerable = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); - - Assert.Throws(typeof(PythonException), () => { - foreach (string item in stringEnumerable) - { - var x = item; - } - }); - Assert.Throws(typeof(PythonException), () => { - stringEnumerable.Count(); - }); - - Runtime.CheckExceptionOccurred(); - - //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable - var foo = GetPythonIterable(); - var fooType = foo.GetPythonType(); - System.Collections.IEnumerable plainEnumerable2 = null; - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); - CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); - - //can convert to any generic ienumerable. If the type is not assignable from the python element - //it will be an exception during TryDecode - Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); - Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); - - Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); - CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); - } - } - - /// - /// "Decodes" only objects of exact type . - /// Result is just the raw proxy to the encoder instance itself. - /// - class ObjectToEncoderInstanceEncoder : IPyObjectEncoder - { - public bool CanEncode(Type type) => type == typeof(T); - public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); - } - - /// - /// Decodes object of specified Python type to the predefined value - /// - /// Type of the - class DecoderReturningPredefinedValue : IPyObjectDecoder - { - public PyObject TheOnlySupportedSourceType { get; } - public TTarget DecodeResult { get; } - - public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) - { - this.TheOnlySupportedSourceType = objectType; - this.DecodeResult = decodeResult; - } - - public bool CanDecode(PyObject objectType, Type targetType) - => objectType.Handle == TheOnlySupportedSourceType.Handle - && targetType == typeof(TTarget); - public bool TryDecode(PyObject pyObj, out T value) - { - if (typeof(T) != typeof(TTarget)) - throw new ArgumentException(nameof(T)); - value = (T)(object)DecodeResult; - return true; - } - } -} +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs + { + [SetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TupleConversionsGeneric() + { + TupleConversionsGeneric, ValueTuple>(); + } + + static void TupleConversionsGeneric() + { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleConversionsObject() + { + TupleConversionsObject, ValueTuple>(); + } + static void TupleConversionsObject() + { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() + { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() + { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) + { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() + { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() + { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) + { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + static string GetIntIterableCommands(string instanceName) + { + var builder = new System.Text.StringBuilder(); + builder.AppendLine(@" +class foo(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter"); + + builder.AppendLine(instanceName + " = foo()"); + return builder.ToString(); + } + + static PyObject GetPythonIterable() + { + var locals = new PyDict(); + using (Py.GIL()) + { + PythonEngine.Exec(GetIntIterableCommands("foo_instance"), null, locals.Handle); + } + + return locals.GetItem("foo_instance"); + } + + [Test] + public void ListDecoderTest() + { + var codec = ListDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + + var pyListType = pyList.GetPythonType(); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //we'd have to copy into a list instance to do this, it would not be lossless. + //lossy converters can be implemented outside of the python.net core library + Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); + + //convert to list of int + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); + CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); + + //convert to list of string. This will not work. + //The ListWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python list can be queried without any conversion, + //the IList will report a Count of 3. + IList stringList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); + Assert.AreEqual(stringList.Count, 3); + Assert.Throws(typeof(PythonException), () => { var x = stringList[0]; }); + + //can't convert python iterable to list (this will require a copy which isn't lossless) + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); + } + + [Test] + public void SequenceDecoderTest() + { + var codec = SequenceDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + //SequenceConverter can only convert to any ICollection + var pyList = new PyList(items.ToArray()); + //it can convert a PyList, since PyList satisfies the python sequence protocol + + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); + + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); + CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); + Assert.AreEqual(3, stringCollection.Count()); + Assert.Throws(typeof(PythonException), () => { + string[] array = new string[3]; + stringCollection.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + //can't convert python iterable to collection (this will require a copy which isn't lossless) + //python iterables do not satisfy the python sequence protocol + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); + + //python tuples do satisfy the python sequence protocol + var pyTuple = new PyObject(Runtime.PyTuple_New(3)); + var pyTupleType = pyTuple.GetPythonType(); + + Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle); + Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle); + Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle); + + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); + CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); + Assert.AreEqual(3, stringCollection2.Count()); + Assert.Throws(typeof(PythonException), () => { + string[] array = new string[3]; + stringCollection2.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + } + + [Test] + public void IterableDecoderTest() + { + var codec = IterableDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + var pyListType = pyList.GetPythonType(); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); + CollectionAssert.AreEqual(plainEnumerable1, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will lead to an empty iterable when decoding. TODO - should it throw? + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); + CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); + + Assert.Throws(typeof(PythonException), () => { + foreach (string item in stringEnumerable) + { + var x = item; + } + }); + Assert.Throws(typeof(PythonException), () => { + stringEnumerable.Count(); + }); + + Runtime.CheckExceptionOccurred(); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); + CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } +} From 17c47a74f91377aaac493f1b5ebbb0c5cc30892f Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sun, 30 Aug 2020 17:27:29 -0500 Subject: [PATCH 15/25] fix compiler error --- src/runtime/CollectionWrappers/ListWrapper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs index 6336151e6..01d13f1c8 100644 --- a/src/runtime/CollectionWrappers/ListWrapper.cs +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -14,7 +14,7 @@ public T this[int index] { get { - var item = Runtime.PyList_GetItem(pyObject.Handle, index); + var item = Runtime.PyList_GetItem(pyObject.Reference, index); var pyItem = new PyObject(item); if (!Converter.ToManaged(pyItem.Handle, typeof(T), out object obj, true)) From e8cc3d82f3efb7720f8d6ca2f352f33adbc972df Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sat, 12 Sep 2020 21:10:34 -0500 Subject: [PATCH 16/25] don't rethrow exception --- src/runtime/CollectionWrappers/SequenceWrapper.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index cd6f9d8fb..d0fd235de 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -43,7 +43,6 @@ public void Clear() if (result == -1) { Runtime.CheckExceptionOccurred(); - throw new Exception("failed to clear sequence"); } } From 080dbc3dd91f6e99792e335d635bd14ca066b677 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 16 Sep 2020 20:04:32 -0500 Subject: [PATCH 17/25] cleanup --- .../CollectionWrappers/IterableWrapper.cs | 50 +------------------ src/runtime/CollectionWrappers/ListWrapper.cs | 24 ++------- .../CollectionWrappers/SequenceWrapper.cs | 1 - 3 files changed, 7 insertions(+), 68 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index 554605025..d174d44db 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -11,55 +11,11 @@ internal class IterableWrapper : IEnumerable public IterableWrapper(PyObject pyObj) { if (pyObj == null) - throw new PythonException(); + throw new ArgumentNullException(); pyObject = pyObj; } - private void propagateIterationException() - { - var err = Runtime.PyErr_Occurred(); - if (err == null) return; - - //remove StopIteration exceptions - if (0 != Runtime.PyErr_ExceptionMatches(Exceptions.StopIteration)) - { - Runtime.PyErr_Clear(); - return; - } - - Runtime.CheckExceptionOccurred(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - PyObject iterObject = null; - using (Py.GIL()) - { - iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); - } - - while (true) - { - IntPtr item = IntPtr.Zero; - using (Py.GIL()) - { - item = Runtime.PyIter_Next(iterObject.Handle); - } - if (item == IntPtr.Zero) break; - - object obj = null; - if (!Converter.ToManaged(item, typeof(object), out obj, true)) - { - Runtime.XDecref(item); - Runtime.CheckExceptionOccurred(); - } - - Runtime.XDecref(item); - yield return obj; - } - - propagateIterationException(); - } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); public IEnumerator GetEnumerator() { @@ -88,8 +44,6 @@ public IEnumerator GetEnumerator() Runtime.XDecref(item); yield return (T)obj; } - - propagateIterationException(); } } } diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs index 01d13f1c8..ec2476370 100644 --- a/src/runtime/CollectionWrappers/ListWrapper.cs +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -16,23 +16,12 @@ public T this[int index] { var item = Runtime.PyList_GetItem(pyObject.Reference, index); var pyItem = new PyObject(item); - - if (!Converter.ToManaged(pyItem.Handle, typeof(T), out object obj, true)) - Runtime.CheckExceptionOccurred(); - - return (T)obj; + return pyItem.As(); } set { - IntPtr pyItem = Converter.ToPython(value, typeof(T)); - if (pyItem == IntPtr.Zero) - { - throw new InvalidCastException( - "cannot cast " + value.ToString() + "to type: " + typeof(T).ToString(), - new PythonException()); - } - - var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem); + var pyItem = value.ToPython(); + var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem.Handle); if (result == -1) Runtime.CheckExceptionOccurred(); } @@ -48,12 +37,9 @@ public void Insert(int index, T item) if (IsReadOnly) throw new InvalidOperationException("Collection is read-only"); - IntPtr pyItem = Converter.ToPython(item, typeof(T)); - if (pyItem == IntPtr.Zero) - throw new PythonException(); + var pyItem = item.ToPython(); - var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem); - Runtime.XDecref(pyItem); + var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem.Handle); if (result == -1) Runtime.CheckExceptionOccurred(); } diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index d0fd235de..b02ef8798 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -18,7 +18,6 @@ public int Count if (size == -1) { Runtime.CheckExceptionOccurred(); - throw new Exception("Unable to get sequence size!"); } return (int)size; From 059ab08d7bbd2effe1f1ed050d97793efc57aa78 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 17 Sep 2020 20:42:34 -0500 Subject: [PATCH 18/25] fixes --- .../CollectionWrappers/IterableWrapper.cs | 21 +++++++------------ src/runtime/converterextensions.cs | 2 +- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index d174d44db..4697bf766 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -21,28 +21,21 @@ public IEnumerator GetEnumerator() { PyObject iterObject = null; using (Py.GIL()) - { iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); - } while (true) { - IntPtr item = IntPtr.Zero; using (Py.GIL()) { - item = Runtime.PyIter_Next(iterObject.Handle); - } - if (item == IntPtr.Zero) break; + var item = Runtime.PyIter_Next(iterObject.Handle); + if (item == IntPtr.Zero) + { + iterObject.Dispose(); + break; + } - object obj = null; - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Runtime.XDecref(item); - Runtime.CheckExceptionOccurred(); + yield return (T)new PyObject(item).AsManagedObject(typeof(T)); } - - Runtime.XDecref(item); - yield return (T)obj; } } } diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index 3a9f18105..b10d0c59f 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -154,7 +154,7 @@ bool TryDecode(IntPtr pyHandle, out object result) #endregion - public static void Reset() + internal static void Reset() { lock (encoders) lock (decoders) From 432c09ec970fddd5006ea9543cd209e1ffc9ab12 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 17 Sep 2020 20:47:19 -0500 Subject: [PATCH 19/25] clean up sequence wrapper --- src/runtime/CollectionWrappers/SequenceWrapper.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index b02ef8798..e0d8058cf 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -54,20 +54,6 @@ public bool Contains(T item) return false; } - private T getItem(int index) - { - IntPtr item = Runtime.PySequence_GetItem(pyObject.Handle, index); - object obj; - - if (!Converter.ToManaged(item, typeof(T), out obj, true)) - { - Runtime.XDecref(item); - Runtime.CheckExceptionOccurred(); - } - - return (T)obj; - } - public void CopyTo(T[] array, int arrayIndex) { var index = 0; From 3043201a048fec63458adb01533d2b94f3b39efe Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sat, 10 Oct 2020 16:58:21 -0500 Subject: [PATCH 20/25] respond to comments --- .../CollectionWrappers/IterableWrapper.cs | 1 + .../CollectionWrappers/SequenceWrapper.cs | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index 4697bf766..a4aa3196d 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -30,6 +30,7 @@ public IEnumerator GetEnumerator() var item = Runtime.PyIter_Next(iterObject.Handle); if (item == IntPtr.Zero) { + Runtime.CheckExceptionOccurred(); iterObject.Dispose(); break; } diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index e0d8058cf..da696e80d 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -56,6 +56,12 @@ public bool Contains(T item) public void CopyTo(T[] array, int arrayIndex) { + if (array == null) + throw new NullReferenceException(); + + if ((array.Length - arrayIndex) < this.Count) + throw new InvalidOperationException("Attempting to copy to an array that is too small"); + var index = 0; foreach (var item in this) { @@ -69,9 +75,15 @@ protected bool removeAt(int index) if (IsReadOnly) throw new NotImplementedException(); if (index >= Count || index < 0) - throw new IndexOutOfRangeException(); + return false; + + var result = Runtime.PySequence_DelItem(pyObject.Handle, index); - return Runtime.PySequence_DelItem(pyObject.Handle, index) != 0; + if (result == 0) + return true; + + Runtime.CheckExceptionOccurred(); + return false; } protected int indexOf(T item) From 57d7779db4b444cd257210d2c688caa493268c8c Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Mon, 19 Oct 2020 21:04:59 -0500 Subject: [PATCH 21/25] fix potential double free --- src/embed_tests/Codecs.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 06bd5ac72..6ffcfe5f1 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -218,13 +218,9 @@ public void SequenceDecoderTest() Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); //python tuples do satisfy the python sequence protocol - var pyTuple = new PyObject(Runtime.PyTuple_New(3)); + var pyTuple = new PyTuple(items.ToArray()); var pyTupleType = pyTuple.GetPythonType(); - Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle); - Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle); - Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle); - Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); From 7fb59158cf8cd361b1e63e5f19695ea21c54d431 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 17 Feb 2021 07:08:48 -0600 Subject: [PATCH 22/25] fix exception --- src/embed_tests/Codecs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 6ffcfe5f1..20c23a781 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -290,7 +290,7 @@ public void IterableDecoderTest() IEnumerable stringEnumerable = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); - Assert.Throws(typeof(PythonException), () => { + Assert.Throws(typeof(InvalidCastException), () => { foreach (string item in stringEnumerable) { var x = item; From 085c6656320dafb4503dad08d897fa0ece2d6f35 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 17 Feb 2021 07:34:37 -0600 Subject: [PATCH 23/25] fix all exceptions --- src/embed_tests/Codecs.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 20c23a781..be8141a69 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -161,7 +161,7 @@ public void ListDecoderTest() IList stringList = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); Assert.AreEqual(stringList.Count, 3); - Assert.Throws(typeof(PythonException), () => { var x = stringList[0]; }); + Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; }); //can't convert python iterable to list (this will require a copy which isn't lossless) var foo = GetPythonIterable(); @@ -204,7 +204,7 @@ public void SequenceDecoderTest() ICollection stringCollection = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); Assert.AreEqual(3, stringCollection.Count()); - Assert.Throws(typeof(PythonException), () => { + Assert.Throws(typeof(InvalidCastException), () => { string[] array = new string[3]; stringCollection.CopyTo(array, 0); }); @@ -241,7 +241,7 @@ public void SequenceDecoderTest() ICollection stringCollection2 = null; Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); Assert.AreEqual(3, stringCollection2.Count()); - Assert.Throws(typeof(PythonException), () => { + Assert.Throws(typeof(InvalidCastException), () => { string[] array = new string[3]; stringCollection2.CopyTo(array, 0); }); @@ -296,7 +296,7 @@ public void IterableDecoderTest() var x = item; } }); - Assert.Throws(typeof(PythonException), () => { + Assert.Throws(typeof(InvalidCastException), () => { stringEnumerable.Count(); }); From bf2d03896079ebbc14bccbe93aedbbc42c077d77 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 18 Feb 2021 15:16:19 -0600 Subject: [PATCH 24/25] respond to PR feedback --- src/embed_tests/Codecs.cs | 24 +------------------ src/runtime/Codecs/IterableDecoder.cs | 2 +- src/runtime/Codecs/ListDecoder.cs | 5 ++-- .../CollectionWrappers/IterableWrapper.cs | 2 +- .../CollectionWrappers/SequenceWrapper.cs | 2 +- 5 files changed, 7 insertions(+), 28 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index be8141a69..266badb9e 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -98,34 +98,12 @@ static void TupleRoundtripGeneric() } } - static string GetIntIterableCommands(string instanceName) - { - var builder = new System.Text.StringBuilder(); - builder.AppendLine(@" -class foo(): - def __init__(self): - self.counter = 0 - def __iter__(self): - return self - def __next__(self): - if self.counter == 3: - raise StopIteration - self.counter = self.counter + 1 - return self.counter"); - - builder.AppendLine(instanceName + " = foo()"); - return builder.ToString(); - } - static PyObject GetPythonIterable() { - var locals = new PyDict(); using (Py.GIL()) { - PythonEngine.Exec(GetIntIterableCommands("foo_instance"), null, locals.Handle); + return PythonEngine.Eval("map(lambda x: x, [1,2,3])"); } - - return locals.GetItem("foo_instance"); } [Test] diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs index 346057238..04e1018d6 100644 --- a/src/runtime/Codecs/IterableDecoder.cs +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -19,7 +19,7 @@ internal static bool IsIterable(Type targetType) internal static bool IsIterable(PyObject objectType) { - return objectType.HasAttr("__iter__"); + return Runtime.PyIter_Check(objectType.Handle); } public bool CanDecode(PyObject objectType, Type targetType) diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs index 23a87f2cc..013f3f3f9 100644 --- a/src/runtime/Codecs/ListDecoder.cs +++ b/src/runtime/Codecs/ListDecoder.cs @@ -15,10 +15,11 @@ private static bool IsList(Type targetType) private static bool IsList(PyObject objectType) { + //TODO accept any python object that implements the sequence and list protocols //must implement sequence protocol to fully implement list protocol - if (!SequenceDecoder.IsSequence(objectType)) return false; + //if (!SequenceDecoder.IsSequence(objectType)) return false; - //returns wheter the type is a list. + //returns wheter the type is a list. return objectType.Handle == Runtime.PyListType; } diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index a4aa3196d..97979b59b 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -12,7 +12,7 @@ public IterableWrapper(PyObject pyObj) { if (pyObj == null) throw new ArgumentNullException(); - pyObject = pyObj; + pyObject = new PyObject(pyObj.Reference); } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs index da696e80d..945019850 100644 --- a/src/runtime/CollectionWrappers/SequenceWrapper.cs +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -14,7 +14,7 @@ public int Count { get { - var size = Runtime.PySequence_Size(pyObject.Handle); + var size = Runtime.PySequence_Size(pyObject.Reference); if (size == -1) { Runtime.CheckExceptionOccurred(); From 07ee9fb986a2138eaf9378099088606573f749ad Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 18 Feb 2021 15:39:32 -0600 Subject: [PATCH 25/25] use hasattr("__iter__") --- src/runtime/Codecs/IterableDecoder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs index 04e1018d6..346057238 100644 --- a/src/runtime/Codecs/IterableDecoder.cs +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -19,7 +19,7 @@ internal static bool IsIterable(Type targetType) internal static bool IsIterable(PyObject objectType) { - return Runtime.PyIter_Check(objectType.Handle); + return objectType.HasAttr("__iter__"); } public bool CanDecode(PyObject objectType, Type targetType)