diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d72a3a0..416b47c53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`). Those marked `[EditorBrowsable(EditorBrowsableState.Never)]`, or declared in a class marked so, keep their static form only +- Types with an implicit conversion to a numeric type support Python arithmetic, comparisons, `float()` and `int()` through that conversion, behaving like a Python `int` for integral conversions and a `float` otherwise, unless they declare or inherit the operator - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs new file mode 100644 index 000000000..d9257ca11 --- /dev/null +++ b/src/embed_tests/TestExtensionMethods.cs @@ -0,0 +1,282 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public enum ExtendedStatus { Open, Closed } + + public interface IExtendedShape + { + double Side { get; } + } + + public class ExtendedModel + { + public string Name { get; set; } = "model"; + + public string Existing() => "member"; + } + + public class DerivedExtendedModel : ExtendedModel + { + } + + public class ExtendedSquare : IExtendedShape + { + public double Side => 2; + } + + public class DisposableExtendedModel : ExtendedModel, IDisposable + { + public void Dispose() + { + } + } + + public static class ExtendedStatusExtensions + { + public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open; + + public static string Describe(this ExtendedStatus status, string prefix, int times = 1) + => string.Concat(Enumerable.Repeat(prefix + status, times)); + } + + public static class ExtendedModelExtensions + { + public static string Describe(this ExtendedModel model, string prefix = "") => prefix + model.Name; + + // collides with the real instance method, which must win + public static string Existing(this ExtendedModel model) => "extension"; + + public static double Area(this IExtendedShape shape) => shape.Side * shape.Side; + + public static int CountModels(this List models) => models.Count; + + public static string Names(this IEnumerable models) => string.Join(",", models.Select(model => model.Name)); + + // targets made only of framework types are not bound, they would bloat every class sharing them + public static int Total(this List values) => values.Sum(); + + public static double SumAll(this double[] values) => values.Sum(); + + public static string Tag(this object value) => "tag"; + + public static int CountItems(this IEnumerable values) => values.Cast().Count(); + + public static bool Release(this IDisposable disposable) => true; + + public static T Identity(this T value) => value; + + public static string Twice(this string value) => value + value; + + // hidden from IntelliSense, so not advertised on the instance either + [EditorBrowsable(EditorBrowsableState.Never)] + public static string Hidden(this ExtendedModel model) => "hidden"; + + [EditorBrowsable(EditorBrowsableState.Advanced)] + public static string Advanced(this ExtendedModel model) => "advanced"; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public static class HiddenExtendedModelExtensions + { + public static string HiddenByClass(this ExtendedModel model) => "hidden by class"; + } + + public class TestExtensionMethods + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from System import Array +from System.Collections.Generic import List +from System.Text import StringBuilder +from Python.EmbeddingTest import * + +def enum_instance(): + return ExtendedStatus.Open.IsOpen(), ExtendedStatus.Open.is_open(), ExtendedStatus.Closed.is_open() + +def enum_with_args(): + return ExtendedStatus.Open.describe('s-', times=2) + +def enum_static_forms(): + return ExtendedStatusExtensions.is_open(ExtendedStatus.Open), ExtendedStatus.is_open(ExtendedStatus.Closed) + +def class_instance(): + return ExtendedModel().describe(), ExtendedModel().describe(prefix='p-') + +def derived_inherits(): + return DerivedExtendedModel().describe('d-') + +def interface_target(): + return ExtendedSquare().area() + +def member_wins(): + return ExtendedModel().existing() + +def closed_generic(): + models = List[ExtendedModel]() + models.Add(ExtendedModel()) + models.Add(ExtendedModel()) + return models.count_models(), models.names() + +def framework_targets(): + values = List[int]() + values.Add(1) + values.Add(2) + array = Array[float]([1.5, 2.5]) + bound = (hasattr(values, 'total'), hasattr(array, 'sum_all'), hasattr(ExtendedModel(), 'tag'), + hasattr(StringBuilder(), 'tag'), hasattr(values, 'count_items'), hasattr(DisposableExtendedModel(), 'release')) + static = (ExtendedModelExtensions.total(values), ExtendedModelExtensions.sum_all(array), + ExtendedModelExtensions.tag(ExtendedModel()), ExtendedModelExtensions.count_items(values), + ExtendedModelExtensions.release(DisposableExtendedModel())) + return bound, static + +def open_generic_not_bound(): + return hasattr(ExtendedModel(), 'identity') + +def editor_browsable(): + model = ExtendedModel() + bound = hasattr(model, 'hidden'), hasattr(model, 'hidden_by_class'), model.advanced() + static = ExtendedModelExtensions.hidden(model), HiddenExtendedModelExtensions.hidden_by_class(model) + return bound, static + +def python_native_not_bound(): + return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab') +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("extension_methods", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void EnumExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.enum_instance(); + Assert.IsTrue((bool)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsFalse((bool)result[2]); + } + + [Test] + public void ExtensionArgumentsFollowTheInstance() + { + using var _ = Py.GIL(); + Assert.AreEqual("s-Opens-Open", (string)module.enum_with_args()); + } + + [Test] + public void StaticFormsKeepWorking() + { + using var _ = Py.GIL(); + var result = module.enum_static_forms(); + Assert.IsTrue((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + + [Test] + public void ClassExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.class_instance(); + Assert.AreEqual("model", (string)result[0]); + Assert.AreEqual("p-model", (string)result[1]); + } + + [Test] + public void DerivedClassInheritsBaseClassExtensions() + { + using var _ = Py.GIL(); + Assert.AreEqual("d-model", (string)module.derived_inherits()); + } + + [Test] + public void InterfaceExtensionIsBoundToImplementingClass() + { + using var _ = Py.GIL(); + Assert.AreEqual(4.0, (double)module.interface_target()); + } + + [Test] + public void RealMemberWinsOverExtension() + { + using var _ = Py.GIL(); + Assert.AreEqual("member", (string)module.member_wins()); + } + + [Test] + public void ClosedGenericTargetOverAUserTypeIsBound() + { + using var _ = Py.GIL(); + var result = module.closed_generic(); + Assert.AreEqual(2, (int)result[0]); + Assert.AreEqual("model,model", (string)result[1]); + } + + [Test] + public void FrameworkOnlyTargetsKeepTheStaticFormOnly() + { + using var _ = Py.GIL(); + var result = module.framework_targets(); + var bound = result[0]; + for (var i = 0; i < 6; i++) + { + Assert.IsFalse((bool)bound[i], $"index {i}"); + } + + var staticForms = result[1]; + Assert.AreEqual(3, (int)staticForms[0]); + Assert.AreEqual(4.0, (double)staticForms[1]); + Assert.AreEqual("tag", (string)staticForms[2]); + Assert.AreEqual(2, (int)staticForms[3]); + Assert.IsTrue((bool)staticForms[4]); + } + + [Test] + public void OpenGenericTargetIsNotBound() + { + using var _ = Py.GIL(); + Assert.IsFalse((bool)module.open_generic_not_bound()); + } + + [Test] + public void EditorBrowsableNeverKeepsTheStaticFormOnly() + { + using var _ = Py.GIL(); + var result = module.editor_browsable(); + Assert.IsFalse((bool)result[0][0]); + Assert.IsFalse((bool)result[0][1]); + Assert.AreEqual("advanced", (string)result[0][2]); + Assert.AreEqual("hidden", (string)result[1][0]); + Assert.AreEqual("hidden by class", (string)result[1][1]); + } + + [Test] + public void PythonNativeTargetKeepsTheStaticForm() + { + using var _ = Py.GIL(); + var result = module.python_native_not_bound(); + Assert.IsFalse((bool)result[0]); + Assert.AreEqual("abab", (string)result[1]); + } + } +} diff --git a/src/embed_tests/TestImplicitNumericOperators.cs b/src/embed_tests/TestImplicitNumericOperators.cs new file mode 100644 index 000000000..cd9de7470 --- /dev/null +++ b/src/embed_tests/TestImplicitNumericOperators.cs @@ -0,0 +1,367 @@ +using System; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class ConvertibleFee + { + public decimal Amount { get; } + + public ConvertibleFee(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleFee fee) => fee.Amount; + } + + public class ConvertibleRatio + { + public double Value { get; } + + public ConvertibleRatio(double value) + { + Value = value; + } + + public static implicit operator double(ConvertibleRatio ratio) => ratio.Value; + } + + public class ConvertibleMoney + { + public decimal Amount { get; } + + public ConvertibleMoney(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleMoney money) => money.Amount; + + // the type's own operator must keep winning over the synthesized one + public static ConvertibleMoney operator +(ConvertibleMoney a, ConvertibleMoney b) => new(a.Amount + b.Amount); + } + + public class ConvertibleCount + { + public long Value { get; } + + public ConvertibleCount(long value) + { + Value = value; + } + + public static implicit operator long(ConvertibleCount count) => count.Value; + } + + public class ConvertibleBase + { + // inherited by ConvertibleDerived, it must keep winning over the synthesized one as in C# + public static string operator +(ConvertibleBase a, ConvertibleBase b) => "base"; + } + + public class ConvertibleDerived : ConvertibleBase + { + public decimal Amount { get; } + + public ConvertibleDerived(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleDerived derived) => derived.Amount; + } + + public class PreciseQuantity : IComparable + { + public decimal Value { get; } + + public PreciseQuantity(decimal value) + { + Value = value; + } + + // equal to 1 once converted to double, only the type's own ordering tells them apart + public static PreciseQuantity AboveOne() => new(1.0000000000000000000000000001m); + + public int CompareTo(PreciseQuantity other) => Value.CompareTo(other.Value); + + public static implicit operator decimal(PreciseQuantity quantity) => quantity.Value; + } + + public class NotConvertible + { + } + + public class TestImplicitNumericOperators + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from Python.EmbeddingTest import * + +fee = ConvertibleFee(2) +other = ConvertibleFee(3) + +def arithmetic(): + return (fee + other, fee + 1, 1 + fee, fee - other, fee - 1, 10 - fee, + fee * other, fee * 2, 2 * fee, other / fee, fee / 2, 10 / fee, -fee) + +def aggregates(): + return sum([fee, other]), max(fee, other) is other, sorted([other, fee])[0] is fee + +def comparisons(): + return (fee < other, fee > other, fee <= other, fee >= other, + fee > 0, fee < 2, fee <= 2, fee >= 2, 1 < fee, 2 >= fee) + +def conversions(): + return float(fee), int(ConvertibleRatio(1.75)), float(ConvertibleRatio(1.75)) + +def own_operator_wins(): + total = ConvertibleMoney(2) + ConvertibleMoney(3) + return type(total).__name__, total.Amount, ConvertibleMoney(2) - ConvertibleMoney(1) + +def not_convertible(): + return hasattr(NotConvertible(), '__add__'), hasattr(NotConvertible(), '__float__') + +count = ConvertibleCount(2) +big = ConvertibleCount(2**53 + 1) + +def integer_arithmetic(): + results = (count + 1, 1 + count, count + count, count - 5, 10 - count, count * 3, 3 * count, -count, count + True) + return all(type(result) is int for result in results), results + +def integer_with_floats(): + results = (count + 0.5, 0.5 + count, count * 1.5, count / 2, count / count, 3 / count) + return all(type(result) is float for result in results), results + +def large_integers(): + exact = 2**53 + 1 + return (int(big) == exact, big > 2**53, big - 1 == 2**53, big * big == exact * exact, + -big == -exact, big >= exact, big + 2**70 == exact + 2**70) + +def int_conversions(): + errors = [] + for value in (float('nan'), float('inf')): + try: + int(ConvertibleRatio(value)) + except Exception as e: + errors.append(type(e).__name__) + return errors, int(ConvertibleRatio(-1.75)), int(ConvertibleFee(12345678901234567891)) == 12345678901234567891 + +def division_by_zero(): + errors = [] + for operation in (lambda: fee / 0, lambda: 1 / ConvertibleFee(0), lambda: count / 0.0, lambda: fee / ConvertibleFee(0)): + try: + operation() + except ZeroDivisionError: + errors.append(True) + return len(errors) + +def other_numeric_operands(): + margin = ConvertibleRatio(0.5) + return fee + 2**40, count + 2**70, fee + margin, count * margin, count < 2**64 + +def unsupported_operands(): + messages = [] + for operation in (lambda: count + 'x', lambda: fee * None, lambda: fee < 'x', lambda: NotConvertible() - fee): + try: + operation() + messages.append(None) + except TypeError as e: + messages.append(str(e)) + return messages + +def inherited_operator_wins(): + derived = ConvertibleDerived(2) + return derived + derived, derived - ConvertibleDerived(1), float(derived) + +def own_ordering_wins(): + one, above = PreciseQuantity(1), PreciseQuantity.AboveOne() + return above > one, one < above, above >= one, one <= above, above < one, sorted([above, one])[0] is one +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("implicit_numeric_operators", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void ArithmeticDelegatesToTheImplicitConversion() + { + using var _ = Py.GIL(); + var result = module.arithmetic(); + var expected = new[] { 5.0, 3.0, 3.0, -1.0, 1.0, 8.0, 6.0, 4.0, 4.0, 1.5, 1.0, 5.0, -2.0 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (double)result[i], $"index {i}"); + } + } + + [Test] + public void AggregatesWork() + { + using var _ = Py.GIL(); + var result = module.aggregates(); + Assert.AreEqual(5.0, (double)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsTrue((bool)result[2]); + } + + [Test] + public void ComparisonsDelegateToTheImplicitConversion() + { + using var _ = Py.GIL(); + var result = module.comparisons(); + var expected = new[] { true, false, true, false, true, false, true, true, true, true }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (bool)result[i], $"index {i}"); + } + } + + [Test] + public void FloatAndIntConversions() + { + using var _ = Py.GIL(); + var result = module.conversions(); + Assert.AreEqual(2.0, (double)result[0]); + Assert.AreEqual(1, (int)result[1]); + Assert.AreEqual(1.75, (double)result[2]); + } + + [Test] + public void TheTypesOwnOperatorsKeepWinning() + { + using var _ = Py.GIL(); + var result = module.own_operator_wins(); + Assert.AreEqual("ConvertibleMoney", (string)result[0]); + Assert.AreEqual(5m, (decimal)result[1]); + // subtraction is not declared by the type, so the synthesized one applies + Assert.AreEqual(1.0, (double)result[2]); + } + + [Test] + public void IntegralConversionsBehaveLikeInts() + { + using var _ = Py.GIL(); + var result = module.integer_arithmetic(); + Assert.IsTrue((bool)result[0], "every result must be an int"); + var expected = new long[] { 3, 3, 4, -3, 8, 6, 6, -2, 3 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (long)result[1][i], $"index {i}"); + } + } + + [Test] + public void IntegralConversionsGiveFloatsWherePythonWould() + { + using var _ = Py.GIL(); + var result = module.integer_with_floats(); + Assert.IsTrue((bool)result[0], "every result must be a float"); + var expected = new[] { 2.5, 2.5, 3.0, 1.0, 1.0, 1.5 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (double)result[1][i], $"index {i}"); + } + } + + [Test] + public void LargeIntegralValuesStayExact() + { + using var _ = Py.GIL(); + var result = module.large_integers(); + for (var i = 0; i < 7; i++) + { + Assert.IsTrue((bool)result[i], $"index {i}"); + } + } + + [Test] + public void IntConversionFollowsPython() + { + using var _ = Py.GIL(); + var result = module.int_conversions(); + CollectionAssert.AreEqual(new[] { "ValueError", "OverflowError" }, ((PyObject)result[0]).As()); + Assert.AreEqual(-1, (int)result[1]); + Assert.IsTrue((bool)result[2], "decimal conversions must truncate exactly"); + } + + [Test] + public void DivisionByZeroRaisesLikePython() + { + using var _ = Py.GIL(); + Assert.AreEqual(4, (int)module.division_by_zero()); + } + + [Test] + public void LargeIntsAndOtherNumbersAreOperands() + { + using var _ = Py.GIL(); + var result = module.other_numeric_operands(); + Assert.AreEqual(2.0 + Math.Pow(2, 40), (double)result[0]); + Assert.AreEqual("1180591620717411303426", result[1].ToString()); + // another type with an implicit numeric conversion is read through its __float__ + Assert.AreEqual(2.5, (double)result[2]); + Assert.AreEqual(1.0, (double)result[3]); + Assert.IsTrue((bool)result[4]); + } + + [Test] + public void UnsupportedOperandsRaisePythonsTypeError() + { + using var _ = Py.GIL(); + var messages = ((PyObject)module.unsupported_operands()).As(); + StringAssert.Contains("unsupported operand type(s) for +", messages[0]); + StringAssert.Contains("unsupported operand type(s) for *", messages[1]); + StringAssert.Contains("'<' not supported", messages[2]); + StringAssert.Contains("unsupported operand type(s) for -", messages[3]); + } + + [Test] + public void InheritedOperatorsKeepWinning() + { + using var _ = Py.GIL(); + var result = module.inherited_operator_wins(); + Assert.AreEqual("base", (string)result[0]); + // subtraction is declared nowhere in the hierarchy, so the synthesized one applies + Assert.AreEqual(1.0, (double)result[1]); + Assert.AreEqual(2.0, (double)result[2]); + } + + [Test] + public void TheTypesOwnOrderingWinsBetweenItsInstances() + { + using var _ = Py.GIL(); + var result = module.own_ordering_wins(); + var expected = new[] { true, true, true, true, false, true }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (bool)result[i], $"index {i}"); + } + } + + [Test] + public void TypesWithoutAConversionAreUntouched() + { + using var _ = Py.GIL(); + var result = module.not_convertible(); + Assert.IsFalse((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index dd7061c2b..da5140d91 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index 3370e4410..76b252652 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -62,6 +62,7 @@ internal static void Initialize() // test/usage after the first init cycle. Clear so the scan runs fresh. assembliesNamesCache.Clear(); assemblies.Clear(); + ExtensionMethods.Clear(); AppDomain domain = AppDomain.CurrentDomain; @@ -356,7 +357,8 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + var types = GetTypes(assembly); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -383,6 +385,8 @@ internal static void ScanAssembly(Assembly assembly) GenericUtil.Register(t); } } + + ExtensionMethods.Register(assembly, types); } public static AssemblyName[] ListAssemblies() diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b88a6a6b6..b4aefacc2 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -415,6 +415,27 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) info = info.Concat(opsImpl.GetMethods(OpsHelper.BindingFlags)).ToArray(); } } + else if (ImplicitNumericOps.IsSupported(type)) + { + // Types with an implicit numeric conversion get Python's number protocol, keeping the operators they declare or inherit + var ops = ImplicitNumericOps.GetMethods(type); + foreach (var op in ops) + { + local.Add(op.Name); + } + info = info.Concat(ops).ToArray(); + } + + // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C# + var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray(); + if (extensionMethods.Length > 0) + { + foreach (var em in extensionMethods) + { + local.Add(em.Name); + } + info = info.Concat(extensionMethods).ToArray(); + } // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs new file mode 100644 index 000000000..dea19f5a0 --- /dev/null +++ b/src/runtime/ExtensionMethods.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Python.Runtime +{ + /// + /// Index of the C# extension methods found in the scanned assemblies, keyed by the type they extend. + /// Classes bind them as members so they are callable from an instance, like C# callers do. + /// + internal static class ExtensionMethods + { + private static readonly ConcurrentDictionary> _methodsByExtendedType = new(); + private static readonly ConcurrentDictionary _isFrameworkAssembly = new(); + private static volatile bool _hasInterfaceTargets; + + // The framework ships thousands of extension methods (LINQ above all); binding them onto + // every collection would bloat each class, so only user assemblies are indexed. + private static readonly string[] _frameworkAssemblyPrefixes = { "System", "Microsoft", "mscorlib", "netstandard", "WindowsBase", "Python.Runtime" }; + + /// + /// Drops the index so a re-initialized runtime rebuilds it while re-scanning the assemblies + /// + internal static void Clear() + { + _methodsByExtendedType.Clear(); + _isFrameworkAssembly.Clear(); + _hasInterfaceTargets = false; + } + + internal static bool IsExtensionMethod(MethodBase method) + { + return method.IsStatic && method.IsDefined(typeof(ExtensionAttribute), false); + } + + /// + /// Indexes the extension methods declared by the given exported types of an assembly + /// + internal static void Register(Assembly assembly, IEnumerable exportedTypes) + { + // The compiler marks the assemblies declaring extension methods, C# itself skips the unmarked ones + if (IsFrameworkAssembly(assembly) || !assembly.IsDefined(typeof(ExtensionAttribute), false)) + { + return; + } + + foreach (var type in exportedTypes) + { + // extension methods can only live in non-generic static classes + if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false) || IsHidden(type)) + { + continue; + } + + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (!IsExtensionMethod(method)) + { + continue; + } + + var extendedType = method.GetParameters()[0].ParameterType; + if (!IsBindableTarget(extendedType) || IsHidden(method)) + { + continue; + } + + if (extendedType.IsInterface) + { + _hasInterfaceTargets = true; + } + + var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List()); + lock (methods) + { + methods.Add(method); + } + } + } + } + + /// + /// Gets the extension methods to bind to the given type: those extending the type itself and the + /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy. + /// + internal static IReadOnlyList GetMethodsToBind(Type type) + { + List? result = null; + AddMethods(type, ref result); + + if (_hasInterfaceTargets && !type.IsInterface) + { + Type[]? baseInterfaces = null; + foreach (var @interface in type.GetInterfaces()) + { + if (!_methodsByExtendedType.ContainsKey(@interface)) + { + continue; + } + + baseInterfaces ??= type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; + if (Array.IndexOf(baseInterfaces, @interface) < 0) + { + AddMethods(@interface, ref result); + } + } + } + + return result ?? (IReadOnlyList)Array.Empty(); + } + + private static void AddMethods(Type extendedType, ref List? result) + { + if (_methodsByExtendedType.TryGetValue(extendedType, out var methods)) + { + lock (methods) + { + (result ??= new List()).AddRange(methods); + } + } + } + + /// + /// Whether the extension methods of the given type should be bound to its instances. The binder cannot + /// infer open generic targets (this T, this IEnumerable<T>) from the instance. Targets made only of + /// framework types (object, IEnumerable, double[], List<int>) are shared by countless classes, which + /// would all be bloated with them (Accord.Math alone extends each primitive array with ~600 methods), + /// so those keep their static form only. + /// + private static bool IsBindableTarget(Type type) + { + if (type.IsByRef || type.ContainsGenericParameters) + { + return false; + } + if (type.HasElementType) + { + return IsBindableTarget(type.GetElementType()!); + } + if (!IsFrameworkAssembly(type.Assembly)) + { + return true; + } + return type.IsGenericType && type.GetGenericArguments().Any(IsBindableTarget); + } + + /// + /// [EditorBrowsable(Never)] hides a member from IntelliSense while keeping it callable, so it is not advertised + /// on the instances either: hidden extension methods, or all those of a hidden class, keep their static form only + /// + private static bool IsHidden(MemberInfo member) + { + return member.IsDefined(typeof(EditorBrowsableAttribute), false) + && member.GetCustomAttribute(false)!.State == EditorBrowsableState.Never; + } + + private static bool IsFrameworkAssembly(Assembly assembly) + { + return _isFrameworkAssembly.GetOrAdd(assembly, a => + { + var name = a.GetName().Name ?? string.Empty; + return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + }); + } + } +} diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 6deec74f8..803ee2f23 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -534,6 +534,17 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe pi = pi.Take(1).ToArray(); } + // An extension method called through an instance gets the instance as its first CLR argument + bool isExtension = inst != null && methodInformation.IsExtension; + if (isExtension) + { + pi = pi.Skip(1).ToArray(); + if (hasNamedArgs) + { + paramNames = paramNames.Skip(1).ToArray(); + } + } + // Must be done after IsOperator section int clrArgCount = pi.Length; @@ -775,6 +786,15 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe } } + if (isExtension) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + margs = new[] { co.inst }.Concat(margs).ToArray(); + } + else continue; + } + var match = new MatchedMethod(kwargsMatched, margs, outs, methodInformation, implicitConversions); if (implicitConversions > 0) { @@ -1238,9 +1258,12 @@ internal class MethodInformation { private ParameterInfo[] _parameterInfo; private string[] _parametersNames; + private bool? _isExtension; public MethodBase MethodBase { get; } + public bool IsExtension => _isExtension ??= ExtensionMethods.IsExtensionMethod(MethodBase); + public bool IsOriginal { get; set; } public ParameterInfo[] ParameterInfo diff --git a/src/runtime/Native/ITypeOffsets.cs b/src/runtime/Native/ITypeOffsets.cs index fb65e76f8..bae8636c5 100644 --- a/src/runtime/Native/ITypeOffsets.cs +++ b/src/runtime/Native/ITypeOffsets.cs @@ -22,6 +22,7 @@ interface ITypeOffsets int nb_true_divide { get; } int nb_and { get; } int nb_int { get; } + int nb_float { get; } int nb_or { get; } int nb_xor { get; } int nb_lshift { get; } diff --git a/src/runtime/Native/TypeOffset.cs b/src/runtime/Native/TypeOffset.cs index c94447f37..76196bcad 100644 --- a/src/runtime/Native/TypeOffset.cs +++ b/src/runtime/Native/TypeOffset.cs @@ -31,6 +31,7 @@ static partial class TypeOffset internal static int nb_or { get; private set; } internal static int nb_xor { get; private set; } internal static int nb_int { get; private set; } + internal static int nb_float { get; private set; } internal static int nb_lshift { get; private set; } internal static int nb_rshift { get; private set; } internal static int nb_remainder { get; private set; } diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 0a24781b8..f5b913a33 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.66")] -[assembly: AssemblyFileVersion("2.0.66")] +[assembly: AssemblyVersion("2.0.68")] +[assembly: AssemblyFileVersion("2.0.68")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 765a0dd2c..57dde5de7 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.66 + 2.0.68 false LICENSE https://github.com/pythonnet/pythonnet diff --git a/src/runtime/Types/OperatorMethod.cs b/src/runtime/Types/OperatorMethod.cs index e905375e0..ae214b05c 100644 --- a/src/runtime/Types/OperatorMethod.cs +++ b/src/runtime/Types/OperatorMethod.cs @@ -56,6 +56,7 @@ static OperatorMethod() ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), ["__int__"] = new SlotDefinition("__int__", TypeOffset.nb_int), + ["__float__"] = new SlotDefinition("__float__", TypeOffset.nb_float), }; ComparisonOpMap = new Dictionary { @@ -136,6 +137,11 @@ static IEnumerable GetOperatorCandidates(Type clrType) .Concat(typeof(FlagEnumOps<>).MakeGenericType(clrType).GetMethods(flags)); } + if (ImplicitNumericOps.IsSupported(clrType)) + { + return clrType.GetMethods(flags).Concat(ImplicitNumericOps.GetMethods(clrType)); + } + return clrType.GetMethods(flags); } diff --git a/src/runtime/Util/ImplicitNumericOps.cs b/src/runtime/Util/ImplicitNumericOps.cs new file mode 100644 index 000000000..d53259f14 --- /dev/null +++ b/src/runtime/Util/ImplicitNumericOps.cs @@ -0,0 +1,334 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Linq.Expressions; +using System.Numerics; +using System.Reflection; + +namespace Python.Runtime +{ + /// + /// Locates the implicit numeric conversion of a CLR type. Python operators never consult implicit + /// conversions, so types declaring one get the number protocol through . + /// + internal static class ImplicitNumericOps + { + private static readonly Type[] NumericTypes = + { + typeof(decimal), typeof(double), typeof(float), typeof(long), typeof(int), typeof(short), typeof(sbyte), + typeof(ulong), typeof(uint), typeof(ushort), typeof(byte) + }; + + internal static MethodInfo? GetConversion(Type type) + { + if (type.IsEnum || type.IsPrimitive || type == typeof(decimal) || type.ContainsGenericParameters) + { + return null; + } + + return type.GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(method => method.Name == "op_Implicit" + && NumericTypes.Contains(method.ReturnType) + && method.GetParameters().Length == 1 + && method.GetParameters()[0].ParameterType == type); + } + + internal static bool IsSupported(Type type) => GetConversion(type) != null; + + /// + /// Gets the operators to graft onto the type. The ones it declares or inherits keep winning, as in C# + /// + internal static MethodInfo[] GetMethods(Type type) + { + var existing = new HashSet(type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy).Select(method => method.Name)); + return typeof(ImplicitNumericOps<>).MakeGenericType(type).GetMethods(OpsHelper.BindingFlags) + .Where(method => !existing.Contains(method.Name)) + .ToArray(); + } + } + + /// + /// The number protocol of a type with an implicit numeric conversion, behaving like the Python number the + /// conversion maps to: an int for integral conversions (exact, results are ints unless Python would give a + /// float) and a float for floating point and decimal ones. Operands that are not numbers return + /// NotImplemented, so Python can try the other operand and raise its usual TypeError. + /// + [Ops] + internal static class ImplicitNumericOps + { + private static readonly MethodInfo Conversion = ImplicitNumericOps.GetConversion(typeof(T)) + ?? throw new InvalidOperationException($"{typeof(T)} has no implicit numeric conversion"); + private static readonly bool IntegralConversion = Type.GetTypeCode(Conversion.ReturnType) is not (TypeCode.Double or TypeCode.Single or TypeCode.Decimal); + private static readonly Func ToDouble = BuildConversion(); + // double to BigInteger throws on NaN and infinity, so floating point conversions go through ToDouble + private static readonly Func? ToInteger = IntegralConversion || Conversion.ReturnType == typeof(decimal) + ? BuildConversion() + : null; + private static readonly bool Comparable = typeof(IComparable).IsAssignableFrom(typeof(T)) || typeof(IComparable).IsAssignableFrom(typeof(T)); + +#pragma warning disable IDE1006 // Naming Styles - must match Python + [ForbidPythonThreads] + public static double __float__(T value) => ToDouble(value); + + [ForbidPythonThreads] + public static object __int__(T value) + { + using (Py.GIL()) + { + if (ToInteger != null) + { + return ToPython(ToInteger(value)); + } + + var number = ToDouble(value); + if (double.IsNaN(number)) + { + throw RaisePython(Exceptions.ValueError, "cannot convert float NaN to integer"); + } + if (double.IsInfinity(number)) + { + throw RaisePython(Exceptions.OverflowError, "cannot convert float infinity to integer"); + } + return ToPython(new BigInteger(number)); + } + } +#pragma warning restore IDE1006 + + public static object op_UnaryNegation(T value) + { + using (Py.GIL()) + { + var number = ToNumber(value); + return number.IsInteger ? ToPython(-number.Integer) : -number.Float; + } + } + + public static object op_Addition(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Add, (x, y) => x + y); + public static object op_Addition(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Add, (x, y) => x + y); + + public static object op_Subtraction(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Subtract, (x, y) => x - y); + public static object op_Subtraction(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Subtract, (x, y) => x - y); + + public static object op_Multiply(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Multiply, (x, y) => x * y); + public static object op_Multiply(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Multiply, (x, y) => x * y); + + // true division: a float even between ints, and like Python it raises ZeroDivisionError instead of returning infinity + public static object op_Division(T a, PyObject b) => Arithmetic(a, b, reverse: false, null, Divide); + public static object op_Division(PyObject a, T b) => Arithmetic(b, a, reverse: true, null, Divide); + + // Python reflects a reversed comparison onto the other operand, so only the forward overloads are needed + public static object op_LessThan(T a, PyObject b) => Compare(a, b, Runtime.Py_LT); + public static object op_GreaterThan(T a, PyObject b) => Compare(a, b, Runtime.Py_GT); + public static object op_LessThanOrEqual(T a, PyObject b) => Compare(a, b, Runtime.Py_LE); + public static object op_GreaterThanOrEqual(T a, PyObject b) => Compare(a, b, Runtime.Py_GE); + + private static object Arithmetic(T self, PyObject? other, bool reverse, + Func? integerOperation, Func floatOperation) + { + // the binder released the GIL to call us + using (Py.GIL()) + { + if (!TryGetOperand(other, out var right)) + { + return Runtime.PyNotImplemented; + } + + var left = ToNumber(self); + if (reverse) + { + (left, right) = (right, left); + } + + if (integerOperation != null && left.IsInteger && right.IsInteger) + { + return ToPython(integerOperation(left.Integer, right.Integer)); + } + return floatOperation(left.AsDouble, right.AsDouble); + } + } + + private static double Divide(double dividend, double divisor) + { + if (divisor == 0) + { + throw RaisePython(Exceptions.ZeroDivisionError, "division by zero"); + } + return dividend / divisor; + } + + private static object Compare(T self, PyObject? other, int op) + { + int order; + using (Py.GIL()) + { + // the type's own ordering wins between its instances, as it did before the number protocol was added + if (Comparable && TryGetInstance(other, out var same)) + { + order = Comparer.Default.Compare(self, same); + } + else + { + if (!TryGetOperand(other, out var right)) + { + return Runtime.PyNotImplemented; + } + + var left = ToNumber(self); + if (!left.IsInteger || !right.IsInteger) + { + // IEEE comparisons, false against NaN as in Python + double x = left.AsDouble, y = right.AsDouble; + return op switch + { + Runtime.Py_LT => x < y, + Runtime.Py_LE => x <= y, + Runtime.Py_GT => x > y, + _ => x >= y, + }; + } + order = left.Integer.CompareTo(right.Integer); + } + } + + return op switch + { + Runtime.Py_LT => order < 0, + Runtime.Py_LE => order <= 0, + Runtime.Py_GT => order > 0, + _ => order >= 0, + }; + } + + private static Number ToNumber(T value) => IntegralConversion ? new Number(ToInteger!(value)) : new Number(ToDouble(value)); + + private static bool TryGetInstance(PyObject? value, out T instance) + { + if (value != null && ManagedType.GetManagedObject(value.Reference) is CLRObject { inst: T clrInstance }) + { + instance = clrInstance; + return true; + } + instance = default!; + return false; + } + + /// + /// Reads a Python operand as a number: instances of , ints (bool included) and + /// floats, and any other object implementing __index__ or __float__ (numpy scalars, other CLR types with + /// an implicit numeric conversion). Requires the GIL. + /// + private static bool TryGetOperand(PyObject? value, out Number number) + { + number = default; + if (value == null) + { + return false; + } + if (TryGetInstance(value, out var instance)) + { + number = ToNumber(instance); + return true; + } + + var reference = value.Reference; + if (Runtime.PyInt_Check(reference)) + { + number = new Number(ToBigInteger(reference)); + return true; + } + if (Runtime.PyObject_TypeCheck(reference, Runtime.PyFloatType)) + { + number = new Number(Runtime.PyFloat_AsDouble(reference)); + return true; + } + if (Runtime.PyObject_HasAttrString(reference, "__index__") != 0) + { + using var integer = Runtime.PyNumber_Long(reference); + if (!integer.IsNull()) + { + number = new Number(ToBigInteger(integer.Borrow())); + return true; + } + } + else if (Runtime.PyObject_HasAttrString(reference, "__float__") != 0) + { + var real = Runtime.PyFloat_AsDouble(reference); + if (real != -1 || !Exceptions.ErrorOccurred()) + { + number = new Number(real); + return true; + } + } + + // a failed probe must not leak, the operand is just not a number + Exceptions.Clear(); + return false; + } + + private static BigInteger ToBigInteger(BorrowedReference integer) + { + var value = Runtime.PyLong_AsLongLong(integer); + if (value.HasValue) + { + return value.Value; + } + + // beyond long, parse its digits + Exceptions.Clear(); + using var digits = Runtime.PyObject_Str(integer); + return BigInteger.Parse(Runtime.GetManagedString(digits.Borrow())!, CultureInfo.InvariantCulture); + } + + /// + /// Converts an integer result to a Python int, which is unbounded. Requires the GIL beyond long. + /// + private static object ToPython(BigInteger value) + { + if (value >= long.MinValue && value <= long.MaxValue) + { + return (long)value; + } + return new PyInt(value.ToString(CultureInfo.InvariantCulture)); + } + + /// + /// Raises a Python exception from a call made by the binder, which restores it to the caller. Requires the GIL. + /// + private static Exception RaisePython(PyObject type, string message) + { + Exceptions.SetError(type.Reference, message); + return PythonException.ThrowLastAsClrException(); + } + + private static Func BuildConversion() + { + var value = Expression.Parameter(typeof(T), "value"); + var body = Expression.Convert(Expression.Call(Conversion, value), typeof(TResult)); + return Expression.Lambda>(body, value).Compile(); + } + + private readonly struct Number + { + public readonly bool IsInteger; + public readonly BigInteger Integer; + public readonly double Float; + + public Number(BigInteger value) + { + IsInteger = true; + Integer = value; + Float = 0; + } + + public Number(double value) + { + IsInteger = false; + Integer = default; + Float = value; + } + + public double AsDouble => IsInteger ? (double)Integer : Float; + } + } +}