diff --git a/README.md b/README.md index c46d24b..5005c2a 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,25 @@ in addition to being usable directly as a `javax.script.ScriptEngineFactory`, it also provides some functionality on top, such as the ability to generate lines of script code based on SciJava events. +## Code completion + +The Script Editor completes code in Jython scripts: Java classes, their +members and constructors (with their overloads), imports, the script's `#@` +parameters and, in a live interpreter, its variables. This is Jython's own +completion, which knows Java exactly, by `scijava-code-lsp-jython`'s language +server; this plugin shows it the scripts as plain Jython (`JythonDialect`), +their parameters declared. + +A Python language server (jedi-language-server, offered by +`scijava-code-lsp-jedi`, which runs it in a small environment of its own, +built the first time it is needed) adds its completions too: for Python +modules, and for Java classes, from Python type stubs (`.pyi`) of the Java +packages the script imports, written in the background +to `~/.cache/scijava/jython-stubs` (or `-Dscijava.jython.stubs=`). The +server analyzes the script as CPython (Jython's standard library is Python +2's, but mostly the same); so it is asked for completions only, not for +problems. + For a complete list of scripting languages available as part of the SciJava platform, see the [Scripting](https://github.com/scijava/scijava-common/wiki/Scripting) page on diff --git a/pom.xml b/pom.xml index 8ef0f03..16d6bdf 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,14 @@ org.scijava.plugins.scripting.jython + 2.101.0 + 0.1.0-SNAPSHOT + 0.1.0-SNAPSHOT + + 2.14.0 + + 2.19.2 + bsd_2 SciJava developers. **/script_templates/** @@ -98,6 +106,18 @@ org.scijava scijava-common + + org.scijava + scijava-code-lsp + ${scijava-code-lsp.version} + + + org.scijava + scijava-code-lsp-jython + ${scijava-code-lsp-jython.version} + + test + @@ -112,4 +132,5 @@ test + diff --git a/src/main/java/org/scijava/plugins/scripting/jython/completion/JythonDialect.java b/src/main/java/org/scijava/plugins/scripting/jython/completion/JythonDialect.java new file mode 100644 index 0000000..e3e703e --- /dev/null +++ b/src/main/java/org/scijava/plugins/scripting/jython/completion/JythonDialect.java @@ -0,0 +1,248 @@ +/*- + * #%L + * JSR-223-compliant Jython scripting language plugin. + * %% + * Copyright (C) 2020 - 2025 SciJava developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.scijava.plugins.scripting.jython.completion; + +import java.lang.reflect.Modifier; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.python.core.PyFunction; +import org.python.core.PyJavaType; +import org.python.core.PyModule; +import org.python.core.PyObject; +import org.scijava.code.lsp.ScriptDialect; +import org.scijava.plugin.Plugin; +import org.scijava.plugins.scripting.jython.JythonScriptLanguage; +import org.scijava.script.ScriptLanguage; + +/** + * Describes how to turn Jython scripts into plain Jython code. Scripts see + * their parameters as the Python values Jython converts them to (e.g. + * {@code str} for strings, {@code int} for integers, {@code array.array} for + * arrays), or else as the Java objects themselves (e.g. + * {@code net.imglib2.img.Img}). + *

+ * Each is declared as an instance of its type, in code both Jython's own + * analysis (Python 2) and a Python language server knowing the Java packages + * from stubs (see scijava-code-lsp-jython) read: e.g. + * {@code import net.imglib2.img; image = net.imglib2.img.Img(); name = str()}. + *

+ *

+ * A live interpreter's variables are declared by their values, where those + * say more: its imported Java classes as the classes themselves, its modules + * imported again, its functions as functions, and its other Python values as + * instances of their builtin types. + *

+ * + * @author Gabriel Selzer + */ +@Plugin(type = ScriptDialect.class, name = "Jython") +public class JythonDialect implements ScriptDialect { + + /** Names Python reserves, which cannot be declared. */ + private static final Set KEYWORDS = new HashSet<>(Arrays.asList( + "False", "None", "True", "and", "as", "assert", "async", "await", "break", + "class", "continue", "def", "del", "elif", "else", "except", "exec", + "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "nonlocal", "not", "or", "pass", "print", "raise", "return", "try", + "while", "with", "yield")); + + @Override + public boolean supports(final ScriptLanguage language) { + return language instanceof JythonScriptLanguage; + } + + @Override + public String languageId() { + return "jython"; + } + + @Override + public Header header(final Map variables) { + // Declare each variable, for type reasoning + final List types = new ArrayList<>(); + final List declarations = new ArrayList<>(); + variables.forEach((name, v) -> { + // NB: Names Python reserves cannot be declared. + if (!KEYWORDS.contains(name)) declarations.add(declaration(name, v, + types)); + }); + // And import the packages of those types, before the declarations + final List code = new ArrayList<>(); + final Set hidden = new TreeSet<>(); + for (final String p : packages(types)) { + code.add("import " + p); + // NB: "import a.b" binds a. + hidden.add(p.split("\\.")[0]); + } + code.addAll(declarations); + // Combine into a Header + return new Header(String.join("; ", code), hidden); + } + + /** + * Declares a variable: by its value, if known (e.g. a live interpreter's + * imported Java class, as the class itself), or else as an instance of its + * type. + * + * @param types Where to note the types the declaration refers to (see + * {@link #packages}). + */ + static String declaration(final String name, final Variable v, + final List types) + { + final Object value = v.value().orElse(null); + // A Java class (e.g. after "from java.util import ArrayList"): itself + final Class c = value instanceof PyJavaType ? ((PyJavaType) value) + .getProxyType() : value instanceof Class ? (Class) value : null; + if (c != null) { + if (!isNamed(c)) return name + " = " + any(types); + types.add(c.getName()); + return name + " = " + c.getName().replace('$', '.'); + } + // A Python module (e.g. after "import os"): imported again + if (value instanceof PyModule) { + final PyObject module = ((PyModule) value).__findattr__("__name__"); + if (module != null && module.toString().matches("[\\w.]+")) { + return "import " + module + " as " + name; + } + } + // A function (e.g. after "def f(): ...") + if (value instanceof PyFunction) { + return name + " = lambda *args, **kwargs: None"; + } + // Another Python value (e.g. a dict): an instance of its builtin type + if (value instanceof PyObject) { + final String builtin = pythonType(((PyObject) value).getType() + .getName()); + return name + " = " + (builtin != null ? builtin + "()" : any(types)); + } + // A Java value, or a script parameter: an instance of its type + final String t = parameterType(v.type()); + types.add(t); + return name + " = " + t.replace('$', '.') + "()"; + } + + /** + * A value of any type (noting the module that says so), for variables + * without a type to declare. + */ + private static String any(final List types) { + types.add("typing.Any"); + return "typing.cast(typing.Any, None)"; + } + + /** + * The Python 3 type of a Python 2 builtin one (as the Python server knows + * Python 3's), e.g. {@code str} for {@code unicode}; or null if not a + * builtin one. + */ + private static String pythonType(final String type) { + switch (type) { + case "int": + case "long": + return "int"; + case "unicode": + return "str"; + case "bool": + case "float": + case "str": + case "list": + case "dict": + case "tuple": + case "set": + return type; + default: + return null; + } + } + + /** Whether code can refer to a class by its name. */ + private static boolean isNamed(final Class c) { + if (c.isPrimitive() || c.isArray() || c.isAnonymousClass() || c + .isLocalClass() || c.getName().indexOf('.') < 0) return false; + for (Class t = c; t != null; t = t.getEnclosingClass()) { + if (!Modifier.isPublic(t.getModifiers())) return false; + } + return true; + } + + /** + * @param type the Java type (of a SciJava Script parameter) declared in the + * script + * @return the type of the object given to the script by Jython: a Python + * type for the values Jython converts (e.g. {@code str}), or else + * the Java type itself + */ + static String parameterType(final Class type) { + final Class t = boxed(type); + // NB: As Jython converts them (Py.java2py). + if (t == String.class || t == Character.class) return "str"; + if (t == Integer.class || t == Long.class || t == Short.class || + t == Byte.class || t == BigInteger.class) return "int"; + if (t == Double.class || t == Float.class) return "float"; + if (t == Boolean.class) return "bool"; + if (t.isArray()) return "array.array"; + return t.getName(); + } + + /** + * The packages or modules of the given types that a declaration refers to + * (none for builtins, e.g. {@code str}). + */ + static Set packages(final Collection types) { + final Set packages = new TreeSet<>(); + for (final String t : types) { + if (t.lastIndexOf('.') > 0) packages.add(t.substring(0, t.lastIndexOf( + '.'))); + } + return packages; + } + + /** Maps primitive types to their wrappers. */ + private static Class boxed(final Class type) { + if (!type.isPrimitive()) return type; + if (type == int.class) return Integer.class; + if (type == long.class) return Long.class; + if (type == double.class) return Double.class; + if (type == float.class) return Float.class; + if (type == boolean.class) return Boolean.class; + if (type == char.class) return Character.class; + if (type == short.class) return Short.class; + if (type == byte.class) return Byte.class; + return Object.class; // void + } +} diff --git a/src/test/java/org/scijava/plugins/scripting/jython/completion/JythonDialectTest.java b/src/test/java/org/scijava/plugins/scripting/jython/completion/JythonDialectTest.java new file mode 100644 index 0000000..13d2294 --- /dev/null +++ b/src/test/java/org/scijava/plugins/scripting/jython/completion/JythonDialectTest.java @@ -0,0 +1,222 @@ +/*- + * #%L + * JSR-223-compliant Jython scripting language plugin. + * %% + * Copyright (C) 2020 - 2025 SciJava developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.scijava.plugins.scripting.jython.completion; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.script.ScriptContext; +import javax.script.ScriptEngine; + +import org.eclipse.lsp4j.CompletionItem; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.scijava.Context; +import org.scijava.code.lsp.LanguageServerService; +import org.python.jsr223.PyScriptEngineFactory; +import org.scijava.code.lsp.ScriptDialect.Header; +import org.scijava.code.lsp.ScriptDialect.Variable; +import org.scijava.code.lsp.ScriptSession; +import org.scijava.script.ScriptLanguage; +import org.scijava.script.ScriptService; + +/** + * Tests {@link JythonDialect}: Jython scripts, as plain Jython for + * scijava-code-lsp-jython's server (which its own tests test). + * + * @author Gabriel Selzer + */ +public class JythonDialectTest { + + private Context context; + private ScriptLanguage jython; + + @Before + public void setUp() { + context = new Context(ScriptService.class, LanguageServerService.class); + jython = context.service(ScriptService.class).getLanguageByName("Jython"); + } + + @After + public void tearDown() { + context.dispose(); + } + + @Test + public void testScriptsAreServed() { + // Jython scripts become plain Jython, which scijava-code-lsp-jython + // serves. + final JythonDialect dialect = new JythonDialect(); + assertTrue(dialect.supports(jython)); + assertEquals("jython", dialect.languageId()); + assertTrue(context.service(LanguageServerService.class).supports(jython)); + } + + @Test + public void testParameterType() { + // As Jython converts them: e.g. strings and numbers to Python's. + assertEquals("str", JythonDialect.parameterType(String.class)); + assertEquals("int", JythonDialect.parameterType(int.class)); + assertEquals("int", JythonDialect.parameterType(Long.class)); + assertEquals("float", JythonDialect.parameterType(double.class)); + assertEquals("bool", JythonDialect.parameterType(boolean.class)); + assertEquals("array.array", JythonDialect.parameterType( + double[].class)); + // Or else as the Java objects themselves. + assertEquals("java.math.BigDecimal", JythonDialect.parameterType( + java.math.BigDecimal.class)); + } + + @Test + public void testHeader() { + // Each parameter is declared as an instance of its type, whose package + // is imported. + final Map parameters = new LinkedHashMap<>(); + parameters.put("d", Variable.ofType(java.math.BigDecimal.class)); + parameters.put("n", Variable.ofType(int.class)); + parameters.put("e", Variable.ofType(Map.Entry.class)); + parameters.put("a", Variable.ofType(double[].class)); + final Header header = new JythonDialect().header(parameters); + assertEquals("import array; import java.math; import java.util; " + + "d = java.math.BigDecimal(); n = int(); " + + "e = java.util.Map.Entry(); a = array.array()", header.code()); + // The names the imports bind ("import a.b" binds a) are the header's + // alone. + assertEquals(Arrays.asList("array", "java"), Arrays.asList(header + .hiddenNames().toArray())); + } + + @Test + public void testLiveHeader() throws Exception { + // A live interpreter's variables, by their values. + final ScriptEngine engine = new PyScriptEngineFactory().getScriptEngine(); + engine.put("injected", new java.util.ArrayList()); + engine.eval("import os\n" + // + "from java.util import HashMap\n" + // + "def helper(): pass\n" + // + "pairs = {1: 2}\n"); + final Map live = new LinkedHashMap<>(); + for (final String name : Arrays.asList("injected", "os", "HashMap", + "helper", "pairs")) + { + final Object value = engine.get(name); + live.put(name, Variable.ofValue(value, value.getClass())); + } + final Header header = new JythonDialect().header(live); + assertEquals("import java.util; " + // + "injected = java.util.ArrayList(); " + // + "import os as os; " + // + "HashMap = java.util.HashMap; " + // + "helper = lambda *args, **kwargs: None; " + // + "pairs = dict()", header.code()); + } + + @Test + public void testScriptParameters() { + final String header = "" + // + "#@ String (label=\"Please enter your name\") name\n" + // + "#@output String greeting\n" + // + "#@ int count\n" + // + "#@ NoSuchType mystery\n"; + assertTrue(complete(header + "x = na").contains("name")); + assertTrue(complete(header + "myst").contains("mystery")); + // As the Python values Jython converts them to: e.g. strings are str, + // whose members are Python's, not java.lang.String's. + final List nameMembers = complete(header + "name.upp"); + assertTrue("expected upper among " + nameMembers, nameMembers.contains( + "upper")); + assertTrue(complete(header + "name.toUpp").isEmpty()); + // Outputs are known too, even before the script assigns them. + assertTrue(complete(header + "greeting.st").contains("strip")); + // Numbers are int, not java.lang.Integer. + assertFalse(complete(header + "count.intV").contains("intValue")); + // Packages imported only to declare those are not completed. + assertFalse(complete(header + "j").contains("java")); + } + + + @Test + public void testAssignmentShadowsScriptParameter() { + final List texts = complete( + "#@ String name\nfrom java.util import ArrayList\nname = ArrayList()\nname.ensureCap"); + assertTrue("expected ArrayList member among " + texts, texts.contains( + "ensureCapacity")); + } + + + @Test + public void testInterpreterBindings() throws Exception { + // Code typed into a running interpreter: its variables. + final ScriptEngine engine = new PyScriptEngineFactory().getScriptEngine(); + engine.put("injected", new java.util.ArrayList()); + engine.eval("from java.util import Collections, HashMap\n" + + "m = HashMap()\n" + + "def helper(): pass\n" + + "import os.path as osp\n"); + final ScriptContext live = engine.getContext(); + assertTrue(complete("m.putIfAb", live).contains("putIfAbsent")); + assertTrue(complete("injected.si", live).contains("size")); + assertTrue(complete("x = hel", live).contains("helper")); + // An imported Java class offers its static members. + assertTrue(complete("Collections.emptyL", live).contains("emptyList")); + // A module bound under another name is the module itself. + assertTrue(complete("osp.joi", live).contains("join")); + } + + // -- Helper methods -- + + private List complete(final String code) { + return complete(code, null); + } + + /** Completions at the end of the code, typed into the given interpreter. */ + private List complete(final String code, final ScriptContext live) { + final LanguageServerService servers = context.service( + LanguageServerService.class); + try (ScriptSession session = live == null ? new ScriptSession(servers, + jython) : new ScriptSession(servers, jython, live)) + { + return session.completion(code, code.length()).get(30, TimeUnit.SECONDS) + .getItems().stream().map(CompletionItem::getLabel).collect(Collectors + .toList()); + } + catch (final Exception exc) { + throw new RuntimeException(exc); + } + } +}