From 6356145bc12aa379eceabb9f32c9723a35cb3642 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Fri, 15 May 2026 23:17:58 +0530 Subject: [PATCH 01/10] Fix SHM truncation handling --- concore.hpp | 28 ++++++++--- tests/test_shm_abort.py | 106 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 tests/test_shm_abort.py diff --git a/concore.hpp b/concore.hpp index e3fb5e1..e6d4e68 100644 --- a/concore.hpp +++ b/concore.hpp @@ -40,6 +40,10 @@ class Concore{ string inpath = "./in"; string outpath = "./out"; + // Shared memory segment size in bytes. + // Increase this constant if your payloads exceed 4096 bytes. + // All nodes in a study must be compiled with the same SHM_SIZE. + // Payloads >= SHM_SIZE throw std::runtime_error (no silent truncation). static constexpr size_t SHM_SIZE = 4096; int shmId_create = -1; @@ -683,9 +687,13 @@ class Concore{ outfile<= SHM_SIZE) { - std::cerr << "ERROR: write_SM payload (" << result.size() - << " bytes) exceeds " << SHM_SIZE - 1 - << "-byte shared memory limit. Data truncated!" << std::endl; + throw std::runtime_error( + "concore SHM write failed: payload (" + + std::to_string(result.size()) + + " bytes) exceeds SHM_SIZE (" + + std::to_string(SHM_SIZE) + + "). Aborting. No data written. Increase SHM_SIZE in concore.hpp." + ); } std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); sharedData_create[SHM_SIZE - 1] = '\0'; @@ -711,15 +719,19 @@ class Concore{ void write_SM(int port, string name, string val, int delta=0){ chrono::milliseconds timespan((int)(2000*delay)); this_thread::sleep_for(timespan); + if (val.size() >= SHM_SIZE) { + throw std::runtime_error( + "concore SHM write failed: payload (" + + std::to_string(val.size()) + + " bytes) exceeds SHM_SIZE (" + + std::to_string(SHM_SIZE) + + "). Aborting. No data written. Increase SHM_SIZE in concore.hpp." + ); + } try { if(shmId_create != -1){ if (sharedData_create == nullptr) throw 506; - if (val.size() >= SHM_SIZE) { - std::cerr << "ERROR: write_SM payload (" << val.size() - << " bytes) exceeds " << SHM_SIZE - 1 - << "-byte shared memory limit. Data truncated!" << std::endl; - } std::strncpy(sharedData_create, val.c_str(), SHM_SIZE - 1); sharedData_create[SHM_SIZE - 1] = '\0'; } diff --git a/tests/test_shm_abort.py b/tests/test_shm_abort.py new file mode 100644 index 0000000..b6a0eb6 --- /dev/null +++ b/tests/test_shm_abort.py @@ -0,0 +1,106 @@ +import shutil +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parent.parent + +pytestmark = pytest.mark.skipif( + shutil.which("g++") is None, + reason="g++ not available", +) + + +@pytest.fixture(autouse=True) +def _skip_windows(): + if sys.platform == "win32": + pytest.skip("SHM requires POSIX") + + +def _compile_and_run(payload_size): + with tempfile.TemporaryDirectory(prefix="concore_shm_test_") as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "concore.oport").write_text('{"1": "1"}', encoding="utf-8") + + source_file = temp_path / "shm_abort_test.cpp" + binary_file = temp_path / "shm_abort_test" + source_file.write_text( + textwrap.dedent( + f''' + #include "concore.hpp" + #include + #include + + int main() {{ + try {{ + Concore concore; + concore.delay = 0; + concore.simtime = 0; + std::string payload({payload_size}, 'a'); + concore.write(1, "payload", payload); + return 0; + }} catch (const std::exception& error) {{ + std::cerr << error.what() << std::endl; + return 1; + }} + }} + ''' + ).lstrip(), + encoding="utf-8", + ) + + compile_result = subprocess.run( + [ + "g++", + "-std=c++17", + "-I", + str(REPO_ROOT), + "-o", + str(binary_file), + str(source_file), + ], + capture_output=True, + text=True, + timeout=60, + cwd=temp_path, + ) + if compile_result.returncode != 0: + pytest.fail(f"g++ compile failed:\n{compile_result.stderr}") + + return subprocess.run( + [str(binary_file)], + capture_output=True, + text=True, + timeout=10, + cwd=temp_path, + ) + + +def test_oversized_payload_throws(): + result = _compile_and_run(5000) + assert result.returncode != 0 + assert "Aborting" in result.stderr + assert "truncated" not in result.stderr.lower() + + +def test_within_limit_succeeds(): + result = _compile_and_run(100) + assert result.returncode == 0 + assert result.stderr == "" + + +def test_exactly_at_limit_throws(): + result = _compile_and_run(4096) + assert result.returncode != 0 + assert "Aborting" in result.stderr + + +def test_one_under_limit_succeeds(): + result = _compile_and_run(4095) + assert result.returncode == 0 + assert result.stderr == "" From b4d4974affddcc0fc407ca7141ea76dda94fde41 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Fri, 15 May 2026 23:28:51 +0530 Subject: [PATCH 02/10] Remove SHM comment block per request --- concore.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/concore.hpp b/concore.hpp index e6d4e68..907e028 100644 --- a/concore.hpp +++ b/concore.hpp @@ -40,10 +40,6 @@ class Concore{ string inpath = "./in"; string outpath = "./out"; - // Shared memory segment size in bytes. - // Increase this constant if your payloads exceed 4096 bytes. - // All nodes in a study must be compiled with the same SHM_SIZE. - // Payloads >= SHM_SIZE throw std::runtime_error (no silent truncation). static constexpr size_t SHM_SIZE = 4096; int shmId_create = -1; From ed4572fd62668822e06751149f7aeb98f5693074 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Fri, 15 May 2026 23:28:52 +0530 Subject: [PATCH 03/10] Remove SHM comment block per request From 9b353b49cd4ed9b7dcaf6641b0c3786104b9eeb3 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Fri, 15 May 2026 23:39:32 +0530 Subject: [PATCH 04/10] Fix lint: format test_shm_abort.py and keep concore.hpp exception behavior --- concore.hpp | 17 +++++++++++++---- tests/test_shm_abort.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/concore.hpp b/concore.hpp index 907e028..ac0901d 100644 --- a/concore.hpp +++ b/concore.hpp @@ -632,8 +632,14 @@ class Concore{ } } - catch(...){ - cout<<"skipping +"< #include @@ -49,7 +49,7 @@ def _compile_and_run(payload_size): return 1; }} }} - ''' + """ ).lstrip(), encoding="utf-8", ) From 654db669db5c00bc83a0ed252ea7161f22eb2d69 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 18 May 2026 01:07:14 +0530 Subject: [PATCH 05/10] ensure SHM write exceptions are logged and rethrown --- concore.hpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/concore.hpp b/concore.hpp index ac0901d..5b8d549 100644 --- a/concore.hpp +++ b/concore.hpp @@ -709,8 +709,11 @@ class Concore{ } } - catch(...){ - cout<<"skipping +"< Date: Mon, 18 May 2026 01:26:05 +0530 Subject: [PATCH 06/10] test: verify SHM path in SHM abort regression --- tests/test_shm_abort.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_shm_abort.py b/tests/test_shm_abort.py index 1f19ae1..68e385f 100644 --- a/tests/test_shm_abort.py +++ b/tests/test_shm_abort.py @@ -34,15 +34,21 @@ def _compile_and_run(payload_size): f""" #include "concore.hpp" #include + #include #include int main() {{ try {{ Concore concore; + std::filesystem::create_directories("out/1"); concore.delay = 0; concore.simtime = 0; std::string payload({payload_size}, 'a'); concore.write(1, "payload", payload); + if (std::filesystem::exists("out/1/payload")) {{ + std::cerr << "write used the file path instead of shared memory" << std::endl; + return 2; + }} return 0; }} catch (const std::exception& error) {{ std::cerr << error.what() << std::endl; @@ -91,7 +97,8 @@ def test_oversized_payload_throws(): def test_within_limit_succeeds(): result = _compile_and_run(100) assert result.returncode == 0 - assert result.stderr == "" + assert "Aborting" not in result.stderr + assert "write used the file path instead of shared memory" not in result.stderr def test_exactly_at_limit_throws(): @@ -103,4 +110,5 @@ def test_exactly_at_limit_throws(): def test_one_under_limit_succeeds(): result = _compile_and_run(4095) assert result.returncode == 0 - assert result.stderr == "" + assert "Aborting" not in result.stderr + assert "write used the file path instead of shared memory" not in result.stderr From 401052bf2b5d665ac0d309c210e0a83548fc0524 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 18 May 2026 01:52:26 +0530 Subject: [PATCH 07/10] feat: improve docker-compose generation with restart, depends_on, network, and zmq mode --- concore_cli/cli.py | 11 +++- concore_cli/commands/build.py | 41 +++++++++----- tests/test_compose_generation.py | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 15 deletions(-) create mode 100644 tests/test_compose_generation.py diff --git a/concore_cli/cli.py b/concore_cli/cli.py index a273675..4b376c9 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -75,8 +75,16 @@ def init(name, template, interactive): is_flag=True, help="Generate docker-compose.yml in output directory (docker type only)", ) -def build(workflow_file, source, output, type, auto_build, compose): +@click.option( + "--zmq", + is_flag=True, + help="Configure compose for ZMQ networking mode (requires --compose)", +) +def build(workflow_file, source, output, type, auto_build, compose, zmq): """Compile a concore workflow into executable scripts""" + if zmq and not compose: + console.print("[red]Error:[/red] --zmq requires --compose") + sys.exit(1) try: build_workflow( workflow_file, @@ -86,6 +94,7 @@ def build(workflow_file, source, output, type, auto_build, compose): auto_build, console, compose=compose, + zmq_mode=zmq, ) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") diff --git a/concore_cli/commands/build.py b/concore_cli/commands/build.py index ee21470..5e29209 100644 --- a/concore_cli/commands/build.py +++ b/concore_cli/commands/build.py @@ -75,9 +75,17 @@ def _parse_docker_run_line(line): } -def _write_docker_compose(output_path): +def _write_docker_compose(output_path, console, zmq_mode=False): run_script = output_path / "run" if not run_script.exists(): + console.print( + "[yellow]Warning:[/yellow] No docker run script found " + f"in {output_path}." + ) + console.print( + "[dim]Tip: run concore build --type docker first, " + "then use --compose[/dim]" + ) return None services = [] @@ -89,15 +97,10 @@ def _write_docker_compose(output_path): if not services: return None - compose_lines = [ - "networks:", - " concore-net:", - " driver: bridge", - "", - "services:", - ] + compose_lines = ["services:"] named_volumes = set() + previous_service_name = None for index, service in enumerate(services, start=1): service_name = re.sub(r"[^A-Za-z0-9_.-]", "-", service["container_name"]).strip( "-." @@ -107,15 +110,11 @@ def _write_docker_compose(output_path): elif not service_name[0].isalpha(): service_name = f"service-{service_name}" - compose_lines.append(f" {_yaml_quote(service_name)}:") + compose_lines.append(f" {service_name}:") compose_lines.append(f" image: {_yaml_quote(service['image'])}") compose_lines.append( f" container_name: {_yaml_quote(service['container_name'])}" ) - compose_lines.append(" restart: on-failure") - compose_lines.append(" networks:") - compose_lines.append(" - concore-net") - if service["volumes"]: compose_lines.append(" volumes:") for volume_spec in service["volumes"]: @@ -123,6 +122,16 @@ def _write_docker_compose(output_path): part1 = volume_spec.split(":")[0] if re.match(r"^[a-zA-Z0-9_-]+$", part1): named_volumes.add(part1) + compose_lines.append(" restart: on-failure") + compose_lines.append(" networks:") + compose_lines.append(" - concore_net") + if zmq_mode: + compose_lines.append(" environment:") + compose_lines.append(" - CONCORE_TRANSPORT=zmq") + if index > 1 and previous_service_name: + compose_lines.append(" depends_on:") + compose_lines.append(f" - {previous_service_name}") + previous_service_name = service_name if named_volumes: compose_lines.append("") @@ -131,6 +140,9 @@ def _write_docker_compose(output_path): compose_lines.append(f" {v}:") compose_lines.append("") + compose_lines.append("networks:") + compose_lines.append(" concore_net:") + compose_lines.append(" driver: bridge") compose_path = output_path / "docker-compose.yml" compose_path.write_text("\n".join(compose_lines), encoding="utf-8") return compose_path @@ -144,6 +156,7 @@ def build_workflow( auto_build, console, compose=False, + zmq_mode=False, ): workflow_path = Path(workflow_file).resolve() source_path = Path(source).resolve() @@ -238,7 +251,7 @@ def build_workflow( ) if compose: - compose_path = _write_docker_compose(output_path) + compose_path = _write_docker_compose(output_path, console, zmq_mode=zmq_mode) if compose_path is not None: console.print( f"[green]✓[/green] Compose file written to [cyan]{compose_path}[/cyan]" diff --git a/tests/test_compose_generation.py b/tests/test_compose_generation.py new file mode 100644 index 0000000..33aff9b --- /dev/null +++ b/tests/test_compose_generation.py @@ -0,0 +1,92 @@ +from concore_cli.commands.build import _write_docker_compose +from rich.console import Console +from pathlib import Path +import pytest + +def _fake_run_script(output_dir, services): + lines = [ + f"docker run --name {s['name']} -v /study:/study {s['image']} &" + for s in services + ] + (Path(output_dir) / "run").write_text("\n".join(lines)) + + +def test_compose_has_restart_policy(tmp_path): + _fake_run_script(tmp_path, [{"name": "node1", "image": "concore/py"}]) + path = _write_docker_compose(tmp_path, Console(quiet=True)) + assert path is not None + content = path.read_text() + assert "restart: on-failure" in content + + +def test_compose_has_network_section(tmp_path): + _fake_run_script(tmp_path, [{"name": "node1", "image": "concore/py"}]) + path = _write_docker_compose(tmp_path, Console(quiet=True)) + content = path.read_text() + assert "concore_net" in content + assert "networks:" in content + + +def test_compose_depends_on_second_service(tmp_path): + _fake_run_script( + tmp_path, + [ + {"name": "controller", "image": "concore/py"}, + {"name": "plant", "image": "concore/cpp"}, + ], + ) + path = _write_docker_compose(tmp_path, Console(quiet=True)) + content = path.read_text() + assert "depends_on" in content + assert "controller" in content + + +def test_compose_first_service_has_no_depends_on(tmp_path): + _fake_run_script( + tmp_path, + [ + {"name": "controller", "image": "concore/py"}, + {"name": "plant", "image": "concore/cpp"}, + ], + ) + path = _write_docker_compose(tmp_path, Console(quiet=True)) + lines = path.read_text().splitlines() + controller_idx = next( + i for i, l in enumerate(lines) if "controller:" in l + ) + plant_idx = next( + i for i, l in enumerate(lines) if "plant:" in l + ) + section = lines[controller_idx:plant_idx] + assert not any("depends_on" in l for l in section) + + +def test_zmq_mode_adds_env(tmp_path): + _fake_run_script(tmp_path, [{"name": "node1", "image": "concore/py"}]) + path = _write_docker_compose( + tmp_path, Console(quiet=True), zmq_mode=True + ) + content = path.read_text() + assert "CONCORE_TRANSPORT=zmq" in content + + +def test_no_zmq_env_in_default_mode(tmp_path): + _fake_run_script(tmp_path, [{"name": "node1", "image": "concore/py"}]) + path = _write_docker_compose( + tmp_path, Console(quiet=True), zmq_mode=False + ) + content = path.read_text() + assert "CONCORE_TRANSPORT" not in content + + +def test_missing_run_script_returns_none(tmp_path): + result = _write_docker_compose(tmp_path, Console(quiet=True)) + assert result is None + + +def test_zmq_without_compose_errors(): + from click.testing import CliRunner + from concore_cli.cli import cli + runner = CliRunner() + result = runner.invoke(cli, ["build", "wf.graphml", "--zmq"]) + assert result.exit_code != 0 From 7ca4672c8056273356bd8e22f96d08cdd9037eab Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 18 May 2026 02:05:16 +0530 Subject: [PATCH 08/10] chore: remove unrelated SHM changes from compose PR --- concore.hpp | 55 ++++++------------- tests/test_shm_abort.py | 114 ---------------------------------------- 2 files changed, 16 insertions(+), 153 deletions(-) delete mode 100644 tests/test_shm_abort.py diff --git a/concore.hpp b/concore.hpp index 5b8d549..e3fb5e1 100644 --- a/concore.hpp +++ b/concore.hpp @@ -632,14 +632,8 @@ class Concore{ } } - catch (const std::exception &e) { - // Surface the error message and rethrow so callers (or the runtime) - // see the failure instead of silently proceeding with truncated data. - std::cerr << e.what() << std::endl; - throw; - } catch (...) { - // Unknown exception: rethrow to avoid silent suppression. - throw; + catch(...){ + cout<<"skipping +"<= SHM_SIZE) { - throw std::runtime_error( - "concore SHM write failed: payload (" + - std::to_string(result.size()) + - " bytes) exceeds SHM_SIZE (" + - std::to_string(SHM_SIZE) + - "). Aborting. No data written. Increase SHM_SIZE in concore.hpp." - ); + std::cerr << "ERROR: write_SM payload (" << result.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; } std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); sharedData_create[SHM_SIZE - 1] = '\0'; @@ -709,11 +696,8 @@ class Concore{ } } - catch (const std::exception &e) { - std::cerr << e.what() << std::endl; - throw; - } catch (...) { - throw; + catch(...){ + cout<<"skipping +"<= SHM_SIZE) { - throw std::runtime_error( - "concore SHM write failed: payload (" + - std::to_string(val.size()) + - " bytes) exceeds SHM_SIZE (" + - std::to_string(SHM_SIZE) + - "). Aborting. No data written. Increase SHM_SIZE in concore.hpp." - ); - } try { if(shmId_create != -1){ if (sharedData_create == nullptr) throw 506; + if (val.size() >= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << val.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } std::strncpy(sharedData_create, val.c_str(), SHM_SIZE - 1); sharedData_create[SHM_SIZE - 1] = '\0'; } else throw 505; } - catch (const std::exception &e) { - std::cerr << e.what() << std::endl; - throw; - } catch (...) { - throw; + catch(...){ + cout<<"skipping +"< - #include - #include - - int main() {{ - try {{ - Concore concore; - std::filesystem::create_directories("out/1"); - concore.delay = 0; - concore.simtime = 0; - std::string payload({payload_size}, 'a'); - concore.write(1, "payload", payload); - if (std::filesystem::exists("out/1/payload")) {{ - std::cerr << "write used the file path instead of shared memory" << std::endl; - return 2; - }} - return 0; - }} catch (const std::exception& error) {{ - std::cerr << error.what() << std::endl; - return 1; - }} - }} - """ - ).lstrip(), - encoding="utf-8", - ) - - compile_result = subprocess.run( - [ - "g++", - "-std=c++17", - "-I", - str(REPO_ROOT), - "-o", - str(binary_file), - str(source_file), - ], - capture_output=True, - text=True, - timeout=60, - cwd=temp_path, - ) - if compile_result.returncode != 0: - pytest.fail(f"g++ compile failed:\n{compile_result.stderr}") - - return subprocess.run( - [str(binary_file)], - capture_output=True, - text=True, - timeout=10, - cwd=temp_path, - ) - - -def test_oversized_payload_throws(): - result = _compile_and_run(5000) - assert result.returncode != 0 - assert "Aborting" in result.stderr - assert "truncated" not in result.stderr.lower() - - -def test_within_limit_succeeds(): - result = _compile_and_run(100) - assert result.returncode == 0 - assert "Aborting" not in result.stderr - assert "write used the file path instead of shared memory" not in result.stderr - - -def test_exactly_at_limit_throws(): - result = _compile_and_run(4096) - assert result.returncode != 0 - assert "Aborting" in result.stderr - - -def test_one_under_limit_succeeds(): - result = _compile_and_run(4095) - assert result.returncode == 0 - assert "Aborting" not in result.stderr - assert "write used the file path instead of shared memory" not in result.stderr From 6d4aff0fe904d9fbd0436678754195697480e9b2 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 18 May 2026 02:17:22 +0530 Subject: [PATCH 09/10] fix: resolve ruff lint errors in test_compose_generation --- tests/test_compose_generation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_compose_generation.py b/tests/test_compose_generation.py index 33aff9b..84e12a7 100644 --- a/tests/test_compose_generation.py +++ b/tests/test_compose_generation.py @@ -1,7 +1,6 @@ from concore_cli.commands.build import _write_docker_compose from rich.console import Console from pathlib import Path -import pytest def _fake_run_script(output_dir, services): lines = [ @@ -52,13 +51,13 @@ def test_compose_first_service_has_no_depends_on(tmp_path): path = _write_docker_compose(tmp_path, Console(quiet=True)) lines = path.read_text().splitlines() controller_idx = next( - i for i, l in enumerate(lines) if "controller:" in l + i for i, line in enumerate(lines) if "controller:" in line ) plant_idx = next( - i for i, l in enumerate(lines) if "plant:" in l + i for i, line in enumerate(lines) if "plant:" in line ) section = lines[controller_idx:plant_idx] - assert not any("depends_on" in l for l in section) + assert not any("depends_on" in line for line in section) def test_zmq_mode_adds_env(tmp_path): From 90a1e833834e0966bb2f427c81941380e831a1dd Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 18 May 2026 02:30:49 +0530 Subject: [PATCH 10/10] ci: install system deps for lxml in test job (ensure lxml builds) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51f238e..fc93695 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + sudo apt-get update && sudo apt-get install -y libxml2-dev libxslt1-dev python3-dev build-essential pip install -r requirements-ci.txt - name: Run tests