diff --git a/test-plans/java-basic-editing.yaml b/test-plans/java-basic-editing.yaml index 0093d73d..c1f8377a 100644 --- a/test-plans/java-basic-editing.yaml +++ b/test-plans/java-basic-editing.yaml @@ -48,6 +48,7 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Java extension has activated and the simple-app project tree is visible in the Explorer sidebar" + skipLlmVerify: true # waitForLanguageServer reads the same status bar text the LLM would inspect verifyProblems: errors: 2 timeout: 120 @@ -140,6 +141,10 @@ steps: path: "~/src/app/App.java" contains: "import java.io.File" timeout: 10 + # Save All has no on-screen change (tab dirty dot clears on the saved + # tab but the active editor isn't the saved file). LLM downgrades on + # before==after by-design. + skipLlmVerify: true # ── Step 8: Rename Symbol (F2) ────────────────────────────── - id: "close-all-before-rename" diff --git a/test-plans/java-debugger.yaml b/test-plans/java-debugger.yaml index a30051c1..92e241b2 100644 --- a/test-plans/java-debugger.yaml +++ b/test-plans/java-debugger.yaml @@ -44,6 +44,7 @@ steps: verify: "Java workspace has loaded; Problems panel shows no errors" verifyProblems: errors: 0 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar timeout: 120 # ── Open App.java ──────────────────────────────────────── diff --git a/test-plans/java-dependency-viewer.yaml b/test-plans/java-dependency-viewer.yaml index 89bbb5c0..258530ce 100644 --- a/test-plans/java-dependency-viewer.yaml +++ b/test-plans/java-dependency-viewer.yaml @@ -28,6 +28,11 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Java workspace has loaded; Explorer shows the project tree and Problems panel is settled" + # waitForLanguageServer is the authoritative deterministic check — the + # status bar can still flicker "Java: Searching... 0%" for background + # indexing right after the LS reports ready, which has historically + # caused LLM screenshot downgrades. Skip LLM here. + skipLlmVerify: true timeout: 120 # ── Open dependency view ───────────────────────────────── diff --git a/test-plans/java-extension-pack.yaml b/test-plans/java-extension-pack.yaml index 7e26ec98..05e7a9d0 100644 --- a/test-plans/java-extension-pack.yaml +++ b/test-plans/java-extension-pack.yaml @@ -30,6 +30,9 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Java workspace has loaded; Explorer shows the project tree and Problems panel is settled" + # waitForLanguageServer is the authoritative deterministic check — + # status-bar background indexing can cause spurious LLM downgrades. + skipLlmVerify: true timeout: 120 # ── Trigger Classpath configuration command ────────────── diff --git a/test-plans/java-fresh-import.yaml b/test-plans/java-fresh-import.yaml index e077e526..b12858fc 100644 --- a/test-plans/java-fresh-import.yaml +++ b/test-plans/java-fresh-import.yaml @@ -45,6 +45,9 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "spring-petclinic project has been imported; Java extension is activated and ready for editing" + # waitForLanguageServer is authoritative — skip LLM screenshot re-check + # (status bar background indexing causes false downgrades). + skipLlmVerify: true timeout: 300 # ── Verify completion ──────────────────────────────────── @@ -67,3 +70,6 @@ steps: verifyCompletion: notEmpty: true waitBefore: 5 + # LLM may downgrade if it sees "Loading..." spinner on cold cache; retry + # gives the LS a warmed cache so the popup is fully rendered. + retries: 1 diff --git a/test-plans/java-gradle-java25.yaml b/test-plans/java-gradle-java25.yaml index 2b8392c9..2462f6e1 100644 --- a/test-plans/java-gradle-java25.yaml +++ b/test-plans/java-gradle-java25.yaml @@ -36,6 +36,7 @@ steps: verifyProblems: errors: 0 timeout: 300 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 2: Open Java file ─────────────────────────────── # wiki: "Open Foo.java, make sure the editing experience is correctly working" @@ -54,6 +55,8 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # ── Step 4: Verify editing ──────────────────────────────── - id: "goto-line" diff --git a/test-plans/java-gradle.yaml b/test-plans/java-gradle.yaml index f6f488cc..02416b46 100644 --- a/test-plans/java-gradle.yaml +++ b/test-plans/java-gradle.yaml @@ -36,6 +36,7 @@ steps: verifyProblems: errors: 0 timeout: 300 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 2: Open Foo.java and verify editing experience ─ # wiki: "Open Foo.java file, make sure the editing experience diff --git a/test-plans/java-maven-java25.yaml b/test-plans/java-maven-java25.yaml index 25c5f444..2a785235 100644 --- a/test-plans/java-maven-java25.yaml +++ b/test-plans/java-maven-java25.yaml @@ -34,6 +34,7 @@ steps: verifyProblems: errors: 0 timeout: 180 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 2: Open Java file ─────────────────────────────── # wiki: "Open Bar.java, make sure the editing experience is correctly working" @@ -52,6 +53,8 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # ── Step 4: Verify editing ──────────────────────────────── - id: "goto-line" diff --git a/test-plans/java-maven-multimodule.yaml b/test-plans/java-maven-multimodule.yaml index ff0e2f89..efe24dec 100644 --- a/test-plans/java-maven-multimodule.yaml +++ b/test-plans/java-maven-multimodule.yaml @@ -32,6 +32,7 @@ steps: action: "waitForLanguageServer" verify: "Multimodule Maven workspace has loaded; the Java extension is initialized for the project with module1 and module2 visible in the Explorer (the Problems panel may briefly show diagnostics that are still being recomputed after import — the verifyProblems checks below pin the final state)" timeout: 180 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 2: Verify module1 Foo.java ────────────────────── # wiki: "make sure the editing experience is correctly working @@ -48,6 +49,8 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # Close module1's tab first so the next `open file Foo.java` request # disambiguates to module2/Foo.java rather than re-focusing the already- @@ -69,3 +72,5 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 diff --git a/test-plans/java-maven-resolve-type.yaml b/test-plans/java-maven-resolve-type.yaml index a136171e..477bc100 100644 --- a/test-plans/java-maven-resolve-type.yaml +++ b/test-plans/java-maven-resolve-type.yaml @@ -49,6 +49,8 @@ steps: action: "waitForLanguageServer" verify: "maven-resolve-type project has been imported; the Java extension is activated and pom.xml is visible in the Explorer" timeout: 180 + # waitForLanguageServer is authoritative — skip LLM screenshot re-check. + skipLlmVerify: true # ── Open Java file ────────────────────────────────────── - id: "open-app" @@ -103,6 +105,10 @@ steps: path: "~/pom.xml" contains: "com.google.code.gson" waitBefore: 2 + # Disk-only insertLineInFile: pom.xml isn't in any editor, so before/after + # screenshots are necessarily identical. LLM always downgrades; verifyFile + # reading from disk is the only meaningful signal. + skipLlmVerify: true # Re-open pom.xml so the AFTER screenshot shows the new # block. Loading fresh from disk avoids any in-memory/disk mismatch. @@ -161,6 +167,10 @@ steps: errors: 0 waitBefore: 20 timeout: 90 + # Maven re-import on a cold cache can take significantly longer than the + # waitBefore window; a single retry (with the LS likely already settled + # by then) recovers without inflating the happy-path wait further. + retries: 1 # After save, the language server publishes diagnostics (status bar updates # to 0 errors, verified deterministically above). However, on Linux runners diff --git a/test-plans/java-maven.yaml b/test-plans/java-maven.yaml index a87e17d3..c2f198bb 100644 --- a/test-plans/java-maven.yaml +++ b/test-plans/java-maven.yaml @@ -30,6 +30,8 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Maven workspace has loaded; the Java extension is initialized and pom.xml is visible in the Explorer (the Problems panel may briefly show diagnostics that are still being recomputed after import)" + # waitForLanguageServer is authoritative — skip LLM screenshot re-check. + skipLlmVerify: true timeout: 120 # ── Step 2: Open Java file and verify editing experience ───────────────── @@ -48,6 +50,8 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # 2c. Verify cursor navigation (goToLine) - id: "goto-line" diff --git a/test-plans/java-new-file-snippet.yaml b/test-plans/java-new-file-snippet.yaml index 8b7ea695..3731340d 100644 --- a/test-plans/java-new-file-snippet.yaml +++ b/test-plans/java-new-file-snippet.yaml @@ -28,6 +28,8 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Java workspace has loaded for the simple-app project; no error notifications visible" + # waitForLanguageServer is authoritative — skip LLM screenshot re-check. + skipLlmVerify: true timeout: 120 # ── Step 9: Create new Java file ───────────────────────────── diff --git a/test-plans/java-pack-help-center-webview.yaml b/test-plans/java-pack-help-center-webview.yaml index 7023d3f6..3d749cd0 100644 --- a/test-plans/java-pack-help-center-webview.yaml +++ b/test-plans/java-pack-help-center-webview.yaml @@ -13,6 +13,13 @@ description: | setup: extension: "redhat.java" + # The Help Center webview lives in vscode-java-pack itself. On scheduled + # runs there is no built VSIX, so the pack must be installed from the + # marketplace; otherwise the `java.welcome` command is unregistered and + # the open-help-center step times out silently. On PR runs the + # build-pack job's VSIX takes precedence (--vsix overrides marketplace). + extensions: + - "vscjava.vscode-java-pack" vscodeVersion: "stable" timeout: 60 settings: diff --git a/test-plans/java-single-file.yaml b/test-plans/java-single-file.yaml index d66b634b..d41867a6 100644 --- a/test-plans/java-single-file.yaml +++ b/test-plans/java-single-file.yaml @@ -31,6 +31,8 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Java extension has activated for the single-file workspace; no error notifications are visible" + # waitForLanguageServer is authoritative — skip LLM screenshot re-check. + skipLlmVerify: true timeout: 120 # ── Step 2: Open Java file ────────────────────────────── @@ -50,6 +52,8 @@ steps: verifyCompletion: notEmpty: true waitBefore: 8 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # ── Step 4: Verify basic editing ──────────────────────────────── - id: "goto-main" diff --git a/test-plans/java-single-no-workspace.yaml b/test-plans/java-single-no-workspace.yaml index eec9557e..87638cbb 100644 --- a/test-plans/java-single-no-workspace.yaml +++ b/test-plans/java-single-no-workspace.yaml @@ -31,6 +31,7 @@ steps: action: "waitForLanguageServer" verify: "Java extension has activated for the single-file (no-workspace) mode; no error notifications visible" timeout: 120 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 2: Verify file is open ────────────────────────────── - id: "verify-file-open" @@ -52,6 +53,8 @@ steps: notEmpty: true waitBefore: 8 timeout: 30 + # Retry once on cold-cache "Loading..." LLM downgrades. + retries: 1 # ── Step 4: Verify editing ──────────────────────────────────── - id: "goto-line" diff --git a/test-plans/java-test-runner.yaml b/test-plans/java-test-runner.yaml index 06021602..ae12f83f 100644 --- a/test-plans/java-test-runner.yaml +++ b/test-plans/java-test-runner.yaml @@ -38,6 +38,7 @@ steps: action: "waitForLanguageServer" verify: "maven-junit workspace has loaded; the Java extension is initialized for the project" timeout: 300 + skipLlmVerify: true # waitForLanguageServer is authoritative; LLM only sees the same status bar # ── Step 1: Open test file so CodeLens can render ─────────── - id: "open-test-file" @@ -52,7 +53,7 @@ steps: # ready — discovery is asynchronous and Test Explorer is initially empty. # On cold-cache CI runners 20s is sometimes too short; bump to 45s. - id: "wait-test-discovery" - action: "wait 45 seconds" + action: "wait 90 seconds" # ── Step 2: Run tests via Java Test Runner palette command ─────── # autotest 0.7.1 ships `openTestExplorer`/`runAllTests` actions wired to @@ -72,6 +73,11 @@ steps: action: "run command Java: Run Tests" verify: "Java: Run Tests command has been invoked from the palette; the Java Test Runner extension has responded (this may show as a Testing view becoming active, a run indicator in the status bar, or an informational notification such as 'No tests found in this file' if discovery is still in progress — all of these indicate the command executed successfully)" waitBefore: 3 + # Test discovery is asynchronous in vscode-java-test; on cold-cache + # CI runners the first Run Tests invocation can land before discovery + # completes ("No tests have been found"). Allow one retry — by the + # second attempt the discovery cache is usually warm. + retries: 1 - id: "wait-test-complete" action: "wait 45 seconds" diff --git a/test-plans/java-webview-migration.yaml b/test-plans/java-webview-migration.yaml index 49aa321a..ec1eeeb1 100644 --- a/test-plans/java-webview-migration.yaml +++ b/test-plans/java-webview-migration.yaml @@ -55,6 +55,10 @@ steps: - id: "ls-ready" action: "waitForLanguageServer" verify: "Maven salut workspace has loaded; the Java extension and the pack webview commands are ready" + # waitForLanguageServer is authoritative — skip LLM screenshot re-check + # (status bar may still show "Java: Searching... 0%" for background + # indexing right after the LS reports ready). + skipLlmVerify: true timeout: 180 # ══════════════════════════════════════════════════════════════════════ @@ -256,6 +260,9 @@ steps: verifyFile: path: "~/.vscode/java-formatter.xml" contains: "CodeFormatterProfile" + # Disk-only insertLineInFile against a file not in any editor. + # before/after screenshots are by-design identical. + skipLlmVerify: true - id: "close-profile-file-before-formatter" action: "run command View: Close All Editors"