feat: mxcli test call — JSON runtime executor against an already-running app

cmd/mxcli/cmd_test_call.go

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/mendixlabs/mxcli/cmd/mxcli/testrunner"
+	"github.com/spf13/cobra"
+)
+
+var testCallCmd = &cobra.Command{
+	Use:   "call <cases.json|->",
+	Short: "Call microflows in the running app with JSON cases (no build, no restart)",
+	Long: `Sends JSON cases to the test endpoint of an app started with
+'mxcli run --local --test-endpoint', runs each microflow inside a transaction
+that is rolled back, and prints pass/fail with expected and actual values.
+
+A case: {"flow": "Mod.Flow", "params": {...}, "setup": [objects created before the call], "expect": [{"path": "$result/Status", "op": "=", "value": "PAID"}], "rollback": true}
+
+params values: a string/number/bool for primitives (enum: the value key), or
+an object: {"$new": "Mod.Entity", "Attr": value, "Mod.Assoc": {nested}} builds
+and commits an object; {"$find": "Mod.Entity[Attr = 'x']"} uses an existing
+one; {"$ref": "name"} reuses a named one ("$name" on the creating object).
+Expect paths: $result, $<param>, then /Member hops; "count:Mod.Entity[xpath]"
+counts rows. Ops: = != < <= > >= contains isnull notnull. "throws": "text"
+expects the call to raise an error containing text.
+
+Rollback covers the main transaction only: separate commits (ExecuteInNewTransaction),
+queued or async work, REST calls, file writes and autonumber counters survive.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		projectPath, _ := cmd.Flags().GetString("project")
+		repeat, _ := cmd.Flags().GetInt("repeat")
+		var r io.Reader = os.Stdin
+		if args[0] != "-" {
+			f, err := os.Open(args[0])
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+				os.Exit(1)
+			}
+			defer f.Close()
+			r = f
+		}
+		cases, err := testrunner.ParseCallCases(r)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		ok, err := testrunner.RunCalls(projectPath, cases, repeat, os.Stdout)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		if !ok {
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	testCallCmd.Flags().Int("repeat", 1, "Run the case list this many times and print the timing of the later rounds")
+	testRunCmd.AddCommand(testCallCmd)
+}

cmd/mxcli/testrunner/call.go

@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package testrunner
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// CallCase is one runtime case for the endpoint's `call` route: a microflow
+// call with parameters built from JSON, expectations on the actual values, and
+// a rollback. Nothing is generated or built; the case is sent as data to the
+// running app.
+type CallCase struct {
+	// Raw is the case exactly as written. It is what is sent, because the order
+	// of the keys in "params" is the order the objects are built in (a later
+	// parameter can $ref an earlier one), and a Go map loses it.
+	Raw json.RawMessage `json:"-"`
+
+	Name     string           `json:"name,omitempty"`
+	Flow     string           `json:"flow"`
+	Params   map[string]any   `json:"params,omitempty"`
+	Setup    []map[string]any `json:"setup,omitempty"`
+	Expect   []map[string]any `json:"expect,omitempty"`
+	Throws   *string          `json:"throws,omitempty"`
+	Rollback *bool            `json:"rollback,omitempty"`
+}
+
+type callCheck struct {
+	Path     string `json:"path"`
+	Op       string `json:"op"`
+	Expected string `json:"expected"`
+	Actual   string `json:"actual"`
+	Pass     bool   `json:"pass"`
+}
+
+type callResponse struct {
+	Pass              bool        `json:"pass"`
+	DurationMicros    int64       `json:"durationMicros"`
+	FlowMicros        int64       `json:"flowMicros"`
+	Result            string      `json:"result"`
+	Error             string      `json:"error"`
+	RollbackRequested bool        `json:"rollbackRequested"`
+	RolledBack        bool        `json:"rolledBack"`
+	Checks            []callCheck `json:"checks"`
+}
+
+// ParseCallCases reads a JSON array of cases, or a single case object.
+func ParseCallCases(r io.Reader) ([]CallCase, error) {
+	body, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	s := strings.TrimSpace(string(body))
+	var raws []json.RawMessage
+	if strings.HasPrefix(s, "[") {
+		err = json.Unmarshal([]byte(s), &raws)
+	} else {
+		raws = []json.RawMessage{json.RawMessage(s)}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading cases: %w", err)
+	}
+	var cases []CallCase
+	for i, raw := range raws {
+		var c CallCase
+		if err := json.Unmarshal(raw, &c); err != nil {
+			return nil, fmt.Errorf("reading case %d: %w", i+1, err)
+		}
+		if c.Flow == "" {
+			return nil, fmt.Errorf("case %d has no \"flow\"", i+1)
+		}
+		c.Raw = raw
+		cases = append(cases, c)
+	}
+	return cases, nil
+}
+
+// RunCalls sends each case to the app hosting the test endpoint for the
+// project, prints pass/fail with expected and actual, and reports whether all
+// passed. repeat > 1 runs the list again and prints only the timing, to show
+// the per-case latency of a warm app.
+func RunCalls(projectPath string, cases []CallCase, repeat int, out io.Writer) (bool, error) {
+	h, err := ReadHandshake(projectPath)
+	if err != nil {
+		return false, err
+	}
+	if err := checkModelNotStale(projectPath); err != nil {
+		return false, err
+	}
+	c := newEndpointClient(h.AppPort, h.Token)
+	if repeat < 1 {
+		repeat = 1
+	}
+	allPass := true
+	for round := 0; round < repeat; round++ {
+		var total time.Duration
+		for _, cs := range cases {
+			b := []byte(cs.Raw)
+			t0 := time.Now()
+			var rr callResponse
+			if err := c.get("call", url.Values{"case": {string(b)}}, &rr); err != nil {
+				return false, err
+			}
+			wall := time.Since(t0)
+			total += wall
+			if round > 0 {
+				if !rr.Pass {
+					allPass = false
+				}
+				continue
+			}
+			name := cs.Name
+			if name == "" {
+				name = cs.Flow
+			}
+			status := "PASS"
+			if !rr.Pass {
+				status = "FAIL"
+				allPass = false
+			}
+			fmt.Fprintf(out, "%s  %s  (%d ms wall, %d ms in app, flow %d ms)\n", status, name,
+				wall.Milliseconds(), rr.DurationMicros/1000, rr.FlowMicros/1000)
+			if rr.Error != "" {
+				fmt.Fprintf(out, "      error: %s\n", rr.Error)
+			}
+			for _, k := range rr.Checks {
+				mark := "ok  "
+				if !k.Pass {
+					mark = "FAIL"
+				}
+				fmt.Fprintf(out, "      %s %s %s %s   actual: %s\n", mark, k.Path, k.Op, k.Expected, k.Actual)
+			}
+			if cs.Rollback == nil || *cs.Rollback {
+				if !rr.RolledBack {
+					fmt.Fprintf(out, "      WARNING: rollback was requested but did not complete\n")
+				}
+			}
+		}
+		if repeat > 1 || round > 0 {
+			fmt.Fprintf(out, "round %d: %d cases in %d ms (%.1f ms per case)\n", round+1, len(cases),
+				total.Milliseconds(), float64(total.Microseconds())/1000/float64(len(cases)))
+		}
+	}
+	return allPass, nil
+}
+
+// checkModelNotStale refuses to call into an app whose model was edited after
+// the build it is running. The running app answers from the model it was built
+// from, so a green result after a model edit would be a stale green. Compares
+// the newest modification under the project's model files (App.mpr, mprcontents,
+// javasource) with the built model (deployment/model/model.mdp).
+func checkModelNotStale(projectPath string) error {
+	dir := filepath.Dir(projectPath)
+	built, err := os.Stat(filepath.Join(dir, "deployment", "model", "model.mdp"))
+	if err != nil {
+		return nil // no build to compare with (Docker, or a moved deployment): nothing to say
+	}
+	var newest time.Time
+	var newestPath string
+	note := func(p string, info fs.FileInfo) {
+		if info.ModTime().After(newest) {
+			newest, newestPath = info.ModTime(), p
+		}
+	}
+	if info, err := os.Stat(projectPath); err == nil {
+		note(projectPath, info)
+	}
+	for _, sub := range []string{"mprcontents", "javasource"} {
+		_ = filepath.WalkDir(filepath.Join(dir, sub), func(p string, d fs.DirEntry, err error) error {
+			if err != nil || d.IsDir() {
+				return nil
+			}
+			if info, err := d.Info(); err == nil {
+				note(p, info)
+			}
+			return nil
+		})
+	}
+	if newest.After(built.ModTime()) {
+		return fmt.Errorf("the model changed after the running app was built (%s, %s after the build)\n"+
+			"  a call now would test the OLD model and could report a stale green.\n"+
+			"  Rebuild and restart first: mxcli run restart -p %s", filepath.Base(newestPath),
+			newest.Sub(built.ModTime()).Round(time.Second), projectPath)
+	}
+	return nil
+}

cmd/mxcli/testrunner/call_test.go

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package testrunner
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestParseCallCases_SingleObject(t *testing.T) {
+	cases, err := ParseCallCases(strings.NewReader(`{"flow": "Mod.Flow"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cases) != 1 || cases[0].Flow != "Mod.Flow" {
+		t.Fatalf("got %+v", cases)
+	}
+}
+
+func TestParseCallCases_Array(t *testing.T) {
+	cases, err := ParseCallCases(strings.NewReader(`[{"flow": "Mod.A"}, {"flow": "Mod.B", "name": "second"}]`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cases) != 2 || cases[0].Flow != "Mod.A" || cases[1].Name != "second" {
+		t.Fatalf("got %+v", cases)
+	}
+}
+
+func TestParseCallCases_MissingFlowRejected(t *testing.T) {
+	if _, err := ParseCallCases(strings.NewReader(`[{"flow": "Mod.A"}, {"name": "no flow here"}]`)); err == nil {
+		t.Fatal("expected an error for a case with no \"flow\"")
+	} else if !strings.Contains(err.Error(), "case 2") {
+		t.Fatalf("error should name the case index, got: %v", err)
+	}
+}
+
+func TestParseCallCases_InvalidJSON(t *testing.T) {
+	if _, err := ParseCallCases(strings.NewReader(`not json`)); err == nil {
+		t.Fatal("expected an error for invalid JSON")
+	}
+}
+
+// touch sets both the content and the mtime of a file, since some filesystems
+// have 1s mtime resolution and these tests compare times taken moments apart.
+func touch(t *testing.T, path string, content []byte, when time.Time) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, content, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(path, when, when); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCheckModelNotStale_NoDeploymentIsNotAnError(t *testing.T) {
+	mpr := tempProject(t)
+	if err := checkModelNotStale(mpr); err != nil {
+		t.Fatalf("no deployment/model/model.mdp at all must not be treated as stale: %v", err)
+	}
+}
+
+func TestCheckModelNotStale_FreshModelPasses(t *testing.T) {
+	mpr := tempProject(t)
+	dir := filepath.Dir(mpr)
+	base := time.Now().Add(-time.Hour)
+	touch(t, mpr, []byte("x"), base)
+	touch(t, filepath.Join(dir, "deployment", "model", "model.mdp"), []byte("model"), base.Add(time.Minute))
+	if err := checkModelNotStale(mpr); err != nil {
+		t.Fatalf("the build is newer than every source file: %v", err)
+	}
+}
+
+// TestCheckModelNotStale_EditedProjectFileRefused is the core safety property:
+// a model edited after the running app's build must refuse a call rather than
+// silently test the old model and report a stale green.
+func TestCheckModelNotStale_EditedProjectFileRefused(t *testing.T) {
+	mpr := tempProject(t)
+	dir := filepath.Dir(mpr)
+	base := time.Now().Add(-time.Hour)
+	touch(t, filepath.Join(dir, "deployment", "model", "model.mdp"), []byte("model"), base)
+	touch(t, mpr, []byte("edited after the build"), base.Add(time.Minute))
+
+	err := checkModelNotStale(mpr)
+	if err == nil {
+		t.Fatal("expected a stale-model error")
+	}
+	if !strings.Contains(err.Error(), "App.mpr") {
+		t.Fatalf("error should name the file that changed, got: %v", err)
+	}
+}
+
+func TestCheckModelNotStale_EditedJavaSourceRefused(t *testing.T) {
+	mpr := tempProject(t)
+	dir := filepath.Dir(mpr)
+	base := time.Now().Add(-time.Hour)
+	touch(t, mpr, []byte("x"), base)
+	touch(t, filepath.Join(dir, "deployment", "model", "model.mdp"), []byte("model"), base)
+	touch(t, filepath.Join(dir, "javasource", "Mod", "MyJavaAction.java"), []byte("edited"), base.Add(time.Minute))
+
+	if err := checkModelNotStale(mpr); err == nil {
+		t.Fatal("a javasource edit after the build must also refuse")
+	}
+}

cmd/mxcli/testrunner/endpoint.go

@@ -224,6 +224,10 @@ com.mendix.core.Core.addRequestHandler("` + endpointPath + `", new com.mendix.ex
             return;
         }
 
+if ("call".equals(path)) {
+            doCall(request, response, out, known);
+            return;
+        }
         if (!"run".equals(path)) {
             response.setStatus(com.mendix.m2ee.api.IMxRuntimeResponse.NOT_FOUND);
             out.write("{\"error\":\"no such route\",\"path\":" + esc(path) + "}");
@@ -308,6 +312,7 @@ com.mendix.core.Core.addRequestHandler("` + endpointPath + `", new com.mendix.ex
         out.write(b.toString());
         out.flush();
     }
+` + endpointCallMethods + endpointCallMethod + `
 });
 
 log.info("MxTest: test endpoint registered at /` + endpointPath + `");