Introduce the ModuleExposer and a test

Port the basic apparatus of modules defined in Java from VSJ and a test. Rationalise ArgParser to eliminate a legacy constructor.
jython · yurem · Jul 31, 2020 · Jul 31, 2020 · Jun 15, 2020 · Jul 30, 2020
commit e81569384db18643a17fb9c548b7b73630efd0ab
diff --git a/core/src/main/java/org/python/core/ArgParser.java b/core/src/main/java/org/python/core/ArgParser.java
diff --git a/core/src/main/java/org/python/core/Exposer.java b/core/src/main/java/org/python/core/Exposer.java
@@ -36,6 +36,7 @@
 import org.python.core.Exposed.PositionalOnly;
 import org.python.core.Exposed.PythonMethod;
 import org.python.core.Exposed.PythonStaticMethod;
+import org.python.core.ModuleDef.MethodDef;
 
 /**
  * An object for tabulating the attributes of classes that define
@@ -67,6 +68,25 @@ protected Exposer() {
     /** @return which {@link ScopeKind} of {@code Exposer} is this? */
     abstract ScopeKind kind();
 
+    /**
+     * On behalf of the given module defined in Java, build a
+     * description of the attributes discovered by introspection of the
+     * class provided.
+     * <p>
+     * Attributes are identified by annotations. (See {@link Exposed}.)
+     *
+     * @param definingClass to introspect for members
+     * @return exposure result
+     * @throws InterpreterError on errors of definition
+     */
+    static ModuleExposer exposeModule(Class<?> definingClass) throws InterpreterError {
+        // Create an instance of Exposer to hold specs, type, etc.
+        ModuleExposer exposer = new ModuleExposer();
+        // Let the exposer control the logic
+        exposer.expose(definingClass);
+        return exposer;
+    }
+
     /**
      * On behalf of the given type defined in Java, build a description
      * of the attributes discovered by introspection of the class (or
@@ -617,14 +637,35 @@ private boolean isDefined() {
         ArgParser getParser() {
             if (parser == null && parameterNames != null
                     && parameterNames.length >= posonlyargcount) {
-                parser = new ArgParser(name, scopeKind, methodKind, varArgsIndex >= 0,
-                        varKeywordsIndex >= 0, posonlyargcount, kwonlyargcount, parameterNames,
-                        regargcount);
+                parser = new ArgParser(name, scopeKind, methodKind, parameterNames, regargcount,
+                        posonlyargcount, kwonlyargcount, varArgsIndex >= 0, varKeywordsIndex >= 0);
                 parser.defaults(defaults).kwdefaults(kwdefaults);
             }
             return parser;
         }
 
+        /**
+         * Produce a method definition from this specification that
+         * references a method handle on the (single) defining method and
+         * the parser created from this specification. This is used in the
+         * construction of a module defined in Java (a {@link ModuleDef}).
+         *
+         * @param lookup authorisation to access methods
+         * @return corresponding method definition
+         * @throws InterpreterError on lookup prohibited
+         */
+        MethodDef getMethodDef(Lookup lookup) throws InterpreterError {
+            assert methods.size() == 1;
+            Method m = methods.get(0);
+            MethodHandle mh;
+            try {
+                mh = lookup.unreflect(m);
+            } catch (IllegalAccessException e) {
+                throw cannotGetHandle(m, e);
+            }
+            return new MethodDef(getParser(), mh);
+        }
+
         /**
          * Add a method implementation. (A test that the signature is
          * acceptable follows when we construct the {@link PyMethodDescr}.)
@@ -1126,9 +1167,9 @@ static class MethodSpec extends CallableSpec {
         @Override
         PyMethodDescr asAttribute(PyType objclass, Lookup lookup) throws InterpreterError {
 
-            ArgParser ap = new ArgParser(name, scopeKind, MethodKind.INSTANCE, varArgsIndex >= 0,
-                    varKeywordsIndex >= 0, posonlyargcount, kwonlyargcount, parameterNames,
-                    regargcount);
+            ArgParser ap = new ArgParser(name, scopeKind, MethodKind.INSTANCE, parameterNames,
+                    regargcount, posonlyargcount, kwonlyargcount, varArgsIndex >= 0,
+                    varKeywordsIndex >= 0);
             ap.defaults(defaults).kwdefaults(kwdefaults);
 
             // Methods have self + this many args:

diff --git a/core/src/main/java/org/python/core/JavaModule.java b/core/src/main/java/org/python/core/JavaModule.java
@@ -0,0 +1,23 @@
+// Copyright (c)2022 Jython Developers.
+// Licensed to PSF under a contributor agreement.
+package org.python.core;
+
+/** Common mechanisms for all Python modules defined in Java. */
+public abstract class JavaModule extends PyModule {
+
+    final ModuleDef definition;
+
+    /**
+     * Construct the base {@code JavaModule} and fill the module
+     * dictionary from the given module definition, which is normally
+     * created during static initialisation of the concrete class
+     * defining the module.
+     *
+     * @param definition of the module
+     */
+    protected JavaModule(ModuleDef definition) {
+        super(definition.name);
+        this.definition = definition;
+        definition.addMembers(this);
+    }
+}
diff --git a/core/src/main/java/org/python/core/ModuleDef.java b/core/src/main/java/org/python/core/ModuleDef.java
@@ -0,0 +1,157 @@
+// Copyright (c)2022 Jython Developers.
+// Licensed to PSF under a contributor agreement.
+package org.python.core;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles.Lookup;
+
+/**
+ * A {@code ModuleDef} is a definition from which instances of a module
+ * may be made. It stands in relation to the Java classes that define
+ * Python modules, somewhat in the way a Python {@code type} object
+ * stands in relation to the Java classes that define Python objects.
+ * <p>
+ * What we most often encounter as "a module", a Python source file, is
+ * actually just a definition from which a module object may be made.
+ * This happens <i>once in each interpreter</i> where the module is
+ * imported. A distinct object, with mutable state, represents that
+ * module in each interpreter. There must therefore be a factory object
+ * that has access to the definition of the module, but is able to
+ * instantiate it (equivalent to executing the body of a module defined
+ * in Python). A {@code ModuleDef} is that factory.
+ * <p>
+ * This initialisation cannot be identified with the static
+ * initialisation of the Java class, since that cannot be repeated, but
+ * must happen per instance. It is useful, however, to have an
+ * intermediate cache of the results of processing the defining Java
+ * class once statically initialised.
+ */
+public class ModuleDef {
+    // Compare CPython PyModuleDef
+
+    /** Name of the module. */
+    final String name;
+
+    /** The Java class defining instances of the module. */
+    final Class<?> definingClass;
+
+    /**
+     * Definitions for the members that appear in the dictionary of
+     * instances of the module named. Instances receive members by copy,
+     * by binding to the module instance (descriptors), or by reference
+     * (if immutable).
+     */
+    private final MethodDef[] methods;
+
+    /**
+     * Create a definition for the module, largely by introspection on
+     * the class and by forming {@code MethodHandle}s on discovered
+     * attributes.
+     *
+     * @param name of the module (e.g. "sys" or "math")
+     * @param lookup authorises access to the defining class.
+     */
+    ModuleDef(String name, Lookup lookup) {
+        this.name = name;
+        this.definingClass = lookup.lookupClass();
+        ModuleExposer exposer = Exposer.exposeModule(definingClass);
+        this.methods = exposer.getMethodDefs(lookup);
+        // XXX ... and for fields.
+        // XXX ... and for types defined in the module maybe? :o
+    }
+
+    /**
+     * Get the method definitions. This method is provided for test use
+     * only. It isn't safe as for public use.
+     *
+     * @return the method definitions
+     */
+    MethodDef[] getMethods() { return methods; }
+
+    /**
+     * Add members defined here to the dictionary of a module instance.
+     *
+     * @param module to populate
+     */
+    void addMembers(JavaModule module) {
+        PyDict d = module.dict;
+        for (MethodDef md : methods) {
+            // Create function by binding to the module
+            PyJavaFunction func = PyJavaFunction.fromParser(
+                    md.argParser, md.handle, module, this.name);
+            d.put(md.argParser.name, func);
+        }
+    }
+
+    /**
+     * A {@code MethodDef} describes a built-in function or method as it
+     * is declared in a Java module. It holds an argument parser and a
+     * handle for calling the method.
+     * <p>
+     * Recall that a module definition may have multiple instances. The
+     * {@code MethodDef} represents the method between the definition of
+     * the module (exposure as a {@link ModuleDef}) and the creation of
+     * actual {@link JavaModule} instances.
+     * <p>
+     * When a method is declared in Java as an instance method of the
+     * module, the {@code MethodDef} that describes it discounts the
+     * {@code self} argument. The {@link PyJavaFunction} created from it
+     * binds the module instance that is its target, so that it is is
+     * correct for a call to that {@code PyJavaFunction}. This is
+     * consistent with CPython.
+     */
+    // Compare CPython struct PyMethodDef
+    static class MethodDef {
+
+        /*
+         * The object here is only superficially similar to the CPython
+         * PyMethodDef: it is not used as a member of descriptors or
+         * methods; extension writers do not declare instances of them.
+         * Instead, we reify the argument information from the
+         * declaration in Java, and associated annotations. In CPython,
+         * this knowledge is present at run-time in the structure of the
+         * code generated by Argument Clinic, incompletely in the flags
+         * of the PyMethodDef, and textually in the signature that
+         * begins the documentation string. We do it by holding an
+         * ArgParser.
+         */
+
+        /**
+         * An argument parser constructed with this {@code MethodDef}
+         * from the description of the signature. Full information on
+         * the signature is available from this structure, and it is
+         * available to parse the arguments to a standard
+         * {@code (Object[], String[])} call. (In simple sub-classes it
+         * is only used to generate error messages once simple checks
+         * fail.)
+         */
+        final ArgParser argParser;
+
+        /**
+         * A handle to the implementation of the function or method.
+         * This is generated by reflecting the same object that
+         * {@link #argParser} describes.
+         */
+        // CPython PyMethodDef: ml_meth
+        final MethodHandle handle;
+
+        /**
+         * Create a {@link MethodDef} of the given kind from the
+         * {@link ArgParser} provided.
+         *
+         * @param argParser parser defining the method
+         * @param meth method handle prepared by sub-class
+         */
+        MethodDef(ArgParser argParser, MethodHandle meth) {
+            this.argParser = argParser;
+            assert meth != null;
+            this.handle = meth;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("%s[%s]", getClass().getSimpleName(),
+                    argParser);
+        }
+    }
+}
diff --git a/core/src/main/java/org/python/core/ModuleExposer.java b/core/src/main/java/org/python/core/ModuleExposer.java
@@ -0,0 +1,88 @@
+// Copyright (c)2022 Jython Developers.
+// Licensed to PSF under a contributor agreement.
+package org.python.core;
+
+import java.lang.invoke.MethodHandles.Lookup;
+import java.lang.reflect.Method;
+
+import org.python.core.Exposed.PythonMethod;
+import org.python.core.Exposed.PythonStaticMethod;
+import org.python.base.InterpreterError;
+import org.python.core.ModuleDef.MethodDef;
+
+/**
+ * A {@code ModuleExposer} provides access to the attributes of a module
+ * defined in Java (a built-in or extension module). These are primarily
+ * the {@link MethodDef}s derived from annotated methods in the defining
+ * class. It is normally obtained by a call to
+ * {@link Exposer#exposeModule(Class)}.
+ */
+class ModuleExposer extends Exposer {
+
+    /**
+     * Construct the {@code ModuleExposer} instance for a particular
+     * module.
+     */
+    ModuleExposer() {}
+
+    /**
+     * Build the result from the defining class.
+     *
+     * @param definingClass to scan for definitions
+     */
+    void expose(Class<?> definingClass) {
+        // Scan the defining class for definitions
+        scanJavaMethods(definingClass);
+        // XXX ... and for fields.
+        // XXX ... and for types defined in the module maybe? :o
+    }
+
+    @Override
+    ScopeKind kind() { return ScopeKind.MODULE; }
+
+    /**
+     * From the methods discovered by introspection of the class, return
+     * an array of {@link MethodDef}s. This array will normally be part
+     * of a {@link ModuleDef} from which the dictionary of each instance
+     * of the module will be created.
+     *
+     * A {@link MethodDef} relies on {@code MethodHandle}, so a lookup
+     * object must be provided with the necessary access to the defining
+     * class.
+     *
+     * @param lookup authorisation to access methods
+     * @return method definitions
+     * @throws InterpreterError on lookup prohibited
+     */
+    MethodDef[] getMethodDefs(Lookup lookup) throws InterpreterError {
+        MethodDef[] a = new MethodDef[methodSpecs.size()];
+        int i = 0;
+        for (CallableSpec ms : methodSpecs) {
+            a[i++] = ms.getMethodDef(lookup);
+        }
+        return a;
+    }
+
+    /**
+     * For a Python module defined in Java, add to {@link specs}, the
+     * methods found in the given defining class and annotated for
+     * exposure.
+     *
+     * @param definingClass to introspect for definitions
+     * @throws InterpreterError on duplicates or unsupported types
+     */
+    @Override
+    void scanJavaMethods(Class<?> definingClass)
+            throws InterpreterError {
+
+        // Collect exposed functions (Java methods)
+        for (Method m : definingClass.getDeclaredMethods()) {
+            PythonMethod a =
+                    m.getDeclaredAnnotation(PythonMethod.class);
+            if (a != null) { addMethodSpec(m, a); }
+            PythonStaticMethod sm =
+                    m.getDeclaredAnnotation(PythonStaticMethod.class);
+            if (sm != null) { addStaticMethodSpec(m, sm); }
+        }
+    }
+}