Stackless issue python#218: Cleanups and test infrastructure for embedded SLP

- Fix test_outside to reset cstack_base and cstack_root. Now the
simulation of a call from outside is complete.
- Add test_slp_embed to test embedding Stackless

Author: Anselm Kruis

Include/slp_structs.h

@@ -144,6 +144,9 @@ typedef struct _cstack {
     struct _cstack *next;
     struct _cstack *prev;
     PY_LONG_LONG serial;
+    /* A borrowed reference to the tasklet, that owns this cstack. NULL after
+     * the stack has been restored. Always NULL for an initial stub.
+     */
     struct _tasklet *task;
     int nesting_level;
     PyThreadState *tstate;
@@ -155,6 +158,8 @@ typedef struct _cstack {
 	 */
     unsigned long exception_list;
 #endif
+    /* The end-address (sic!) of the stack stored in the cstack.
+     */
     intptr_t *startaddr;
     intptr_t stack[
 #if defined(_WINDOWS_) && defined(SLP_SEH32)

Makefile.pre.in

@@ -786,7 +786,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
 Programs/python.o: $(srcdir)/Programs/python.c
 	$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c
 
-Programs/_testembed.o: $(srcdir)/Programs/_testembed.c
+Programs/_testembed.o: $(srcdir)/Programs/_testembed.c $(srcdir)/Stackless/unittests/_testembed_slp.c
 	$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c
 
 Modules/_sre.o: $(srcdir)/Modules/_sre.c $(srcdir)/Modules/sre.h $(srcdir)/Modules/sre_constants.h $(srcdir)/Modules/sre_lib.h

Programs/_testembed.c

@@ -620,6 +620,10 @@ static struct TestCase TestCases[] = {
     { NULL, NULL }
 };
 
+#ifdef STACKLESS
+#include "../Stackless/unittests/_testembed_slp.c"
+#endif
+
 int main(int argc, char *argv[])
 {
     if (argc > 1) {

Stackless/core/stacklesseval.c

@@ -348,17 +348,18 @@ slp_eval_frame(PyFrameObject *f)
         if (ts->st.cstack_base == NULL)
             ts->st.cstack_base = stackref - SLP_CSTACK_GOODGAP;
         if (stackref > ts->st.cstack_base) {
-			PyCStackObject *cst;
+			PyCStackObject *initial_stub;
             retval = climb_stack_and_eval_frame(f);
-			cst = ts->st.initial_stub;
-			/* might be NULL in OOM conditions */
-			if (cst != NULL) {
-                PyCStackObject *other;
+			initial_stub = ts->st.initial_stub;
+			/* cst might be NULL in OOM conditions */
+			if (initial_stub != NULL) {
+                PyCStackObject *cst;
                 register int found = 0;
-                assert(cst->startaddr == ts->st.cstack_base);
-                for (other = cst->next; other != cst; other = other->next) {
-                    if (Py_SIZE(other) != 0 && other->task != NULL && other->tstate == ts) {
-                        assert(other->startaddr == ts->st.cstack_base);
+                assert(initial_stub->startaddr == ts->st.cstack_base);
+                for (cst = initial_stub->next; cst != initial_stub; cst = cst->next) {
+                    if (Py_SIZE(cst) != 0 && cst->task != NULL &&
+                            cst->tstate == ts) {
+                        assert(cst->startaddr == ts->st.cstack_base);
                         found = 1;
                         break;
                     }

Stackless/module/stacklessmodule.c

@@ -969,15 +969,22 @@ test_outside(PyObject *self)
 {
     PyThreadState *ts = PyThreadState_GET();
     PyTaskletObject *stmain = ts->st.main;
-    PyCStackObject *cst = ts->st.initial_stub;
+    PyCStackObject *initial_stub = ts->st.initial_stub;
     PyFrameObject *f = SLP_CURRENT_FRAME(ts);
     int recursion_depth = ts->recursion_depth;
     int nesting_level = ts->st.nesting_level;
+    intptr_t *cstack_base = ts->st.cstack_base;
+    intptr_t *cstack_root = ts->st.cstack_root;
     PyObject *ret = Py_None;
     PY_LONG_LONG jump = ts->st.serial_last_jump;
 
     Py_INCREF(ret);
     ts->st.main = NULL;
+    assert(initial_stub->startaddr == cstack_base);
+    if (ts->interp != _PyRuntime.interpreters.main) {
+        ts->st.cstack_base = NULL;
+        ts->st.cstack_root = NULL;
+    }
     ts->st.initial_stub = NULL;
     ts->st.nesting_level = 0;
     SLP_SET_CURRENT_FRAME(ts, NULL);
@@ -990,9 +997,12 @@ test_outside(PyObject *self)
             break;
         }
     }
+    ts->st.cstack_base = cstack_base;
+    ts->st.cstack_root = cstack_root;
+    initial_stub->startaddr = cstack_base;
     ts->st.main = stmain;
     Py_CLEAR(ts->st.initial_stub);
-    ts->st.initial_stub = cst;
+    ts->st.initial_stub = initial_stub;
     SLP_SET_CURRENT_FRAME(ts, f);
     slp_current_insert(stmain);
     ts->recursion_depth = recursion_depth;

Stackless/unittests/_testembed_slp.c

@@ -0,0 +1,48 @@
+#include <stackless_api.h>
+
+
+static int slp_test_schedule(void)
+{
+    _testembed_Py_Initialize();
+    PyRun_SimpleString(
+"import stackless\n"
+"import sysconfig\n"
+"stackless.tasklet(print)('Hello, World!')\n"
+    );
+    if (!PyStackless_Schedule(Py_None, 0))
+        PyErr_Print();
+    Py_Finalize();
+    return 0;
+}
+
+
+static struct TestCase SlpTestCases[] = {
+    { "slp_schedule", slp_test_schedule },
+    { NULL, NULL }
+};
+
+int main(int argc, char *argv[])
+{
+    int wrapped_main(int, char **);
+
+    if (argc > 1) {
+        for (struct TestCase *tc = SlpTestCases; tc && tc->name; tc++) {
+            if (strcmp(argv[1], tc->name) == 0)
+                return (*tc->func)();
+        }
+        return wrapped_main(argc, argv);
+    }
+
+    /* No match found, or no test name provided, so display usage */
+    wrapped_main(argc, argv);
+    for (struct TestCase *tc = SlpTestCases; tc && tc->name; tc++) {
+        printf("  %s\n", tc->name);
+    }
+
+    /* Non-zero exit code will cause test_embed.py tests to fail.
+       This is intentional. */
+    return -1;
+}
+
+
+#define main wrapped_main

Stackless/unittests/test_outside.py

@@ -69,7 +69,7 @@ def sink():
         def createSource():
             tasklet(source)()
         tasklet(createSource)()  # create source from a different cstate
-        test_outside()
+        test_outside()  # create the source tasklet and run it until it blocks
 
         # now create the sink tasklet
         tasklet(sink)()

Stackless/unittests/test_slp_embed.py

@@ -0,0 +1,39 @@
+# Run the tests in Stackless/unittests/_testembed_slp.c
+# (tests for the Stackless Python embedding APIs)
+"""Test the (Stackless)-Python C-API
+
+Tests not relevant for pure Python code.
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import unittest
+import os
+import sys
+
+from support import test_main  # @UnusedImport
+
+from test.test_embed import EmbeddingTestsMixin
+from test.support import verbose
+
+
+@unittest.skip("Stackless issue 218")
+class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
+    def test_schedule(self):
+        env = dict(os.environ)
+        out, err = self.run_embedded_interpreter("slp_schedule", env=env)
+        if verbose > 1:
+            print()
+            print(out)
+            print(err)
+        expected_output = '\n'.join([
+            "Hello, World!"])
+        # This is useful if we ever trip over odd platform behaviour
+        self.maxDiff = None
+        self.assertEqual(out.strip(), expected_output)
+
+
+if __name__ == "__main__":
+    if not sys.argv[1:]:
+        sys.argv.append('-v')
+    unittest.main()