diff --git a/.azdo/ci-pr.yaml b/.azdo/ci-pr.yaml index 80fc4b4d..178875b0 100644 --- a/.azdo/ci-pr.yaml +++ b/.azdo/ci-pr.yaml @@ -43,7 +43,7 @@ extends: - script: | python -m pip install --upgrade pip python -m pip install flake8 black build diff-cover - python -m pip install -e .[dev] + python -m pip install -e .[dev,async] displayName: 'Install dependencies' - script: | diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 886bc72b..0178b090 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 black build diff-cover - python -m pip install -e .[dev] + python -m pip install -e .[dev,async] - name: Check format with black run: | diff --git a/examples/aio/__init__.py b/examples/aio/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/_auth.py b/examples/aio/_auth.py new file mode 100644 index 00000000..a425321d --- /dev/null +++ b/examples/aio/_auth.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async credential helper for the async example scripts. + +azure-identity's InteractiveBrowserCredential is only available in the sync +namespace (azure.identity), not the async one (azure.identity.aio). This +module wraps the sync credential so it satisfies the AsyncTokenCredential +protocol required by AsyncDataverseClient. + +Usage:: + + from _auth import AsyncInteractiveBrowserCredential + + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(org_url, credential) as client: + ... + finally: + await credential.close() +""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +from azure.identity import InteractiveBrowserCredential + + +class AsyncInteractiveBrowserCredential: + """ + Async wrapper around the sync InteractiveBrowserCredential. + + get_token() is dispatched to a dedicated thread so the event loop stays + free during the browser popup / token exchange. Subsequent calls hit the + in-process token cache and return almost immediately. + """ + + def __init__(self, **kwargs): + self._credential = InteractiveBrowserCredential(**kwargs) + self._executor = ThreadPoolExecutor(max_workers=1) + + async def get_token(self, *scopes, **kwargs): + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + self._executor, + lambda: self._credential.get_token(*scopes, **kwargs), + ) + + async def close(self): + self._executor.shutdown(wait=False) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + await self.close() diff --git a/examples/aio/advanced/__init__.py b/examples/aio/advanced/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/advanced/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/advanced/alternate_keys_upsert.py b/examples/aio/advanced/alternate_keys_upsert.py new file mode 100644 index 00000000..a080975d --- /dev/null +++ b/examples/aio/advanced/alternate_keys_upsert.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Alternate Keys & Upsert Example + +Async equivalent of examples/advanced/alternate_keys_upsert.py. + +Demonstrates the full workflow of creating alternate keys and using +them for upsert operations: +1. Create a custom table with columns +2. Define an alternate key on a column +3. Wait for the key index to become Active +4. Upsert records using the alternate key +5. Verify records were created/updated correctly +6. Clean up + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +# --- Config --- +TABLE_NAME = "new_AltKeyDemo" +KEY_COLUMN = "new_externalid" +KEY_NAME = "new_ExternalIdKey" +BACKOFF_DELAYS = (0, 3, 10, 20, 35) + + +# --- Helpers --- +async def backoff(coro_fn, *, delays=BACKOFF_DELAYS): + """Retry *coro_fn* with exponential-ish backoff on any exception.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" [INFO] Backoff succeeded after {retry_count} retry(s); " f"waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); " f"waited {total_delay}s total.") + raise last + + +async def wait_for_key_active(client, table, key_name, max_wait=120): + """Poll get_alternate_keys until the key status is Active.""" + import time + + start = time.time() + while time.time() - start < max_wait: + keys = await client.tables.get_alternate_keys(table) + for k in keys: + if k.schema_name == key_name: + print(f" Key status: {k.status}") + if k.status == "Active": + return k + if k.status == "Failed": + raise RuntimeError(f"Alternate key index failed: {k.schema_name}") + await asyncio.sleep(5) + raise TimeoutError(f"Key {key_name} did not become Active within {max_wait}s") + + +# --- Main --- +async def main(): + """Run the async alternate-keys & upsert E2E walkthrough.""" + print("PowerPlatform Dataverse Client - Async Alternate Keys & Upsert Example") + print("=" * 70) + print("This script demonstrates:") + print(" - Creating a custom table with columns") + print(" - Defining an alternate key on a column") + print(" - Waiting for the key index to become Active") + print(" - Upserting records via alternate key (create + update)") + print(" - Verifying records and listing keys") + print(" - Cleaning up (delete key, delete table)") + print("=" * 70) + + entered = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not entered: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = entered.rstrip("/") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + + # ------------------------------------------------------------------ + # Step 1: Create table (skip if already exists) + # ------------------------------------------------------------------ + print("\n1. Creating table...") + table_info = await client.tables.get(TABLE_NAME) + if table_info: + print(f" Table already exists: {TABLE_NAME} (skipped)") + else: + table_info = await backoff( + lambda: client.tables.create( + TABLE_NAME, + columns={ + KEY_COLUMN: "string", + "new_ProductName": "string", + "new_Price": "decimal", + }, + ) + ) + print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") + await asyncio.sleep(10) # Wait for metadata propagation + + # ------------------------------------------------------------------ + # Step 2: Create alternate key (skip if already exists) + # ------------------------------------------------------------------ + print("\n2. Creating alternate key...") + existing_keys = await client.tables.get_alternate_keys(TABLE_NAME) + existing_key = next((k for k in existing_keys if k.schema_name == KEY_NAME), None) + if existing_key: + print(f" Alternate key already exists: {KEY_NAME} (skipped)") + else: + key_info = await backoff( + lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()]) + ) + print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") + + # ------------------------------------------------------------------ + # Step 3: Wait for key to become Active + # ------------------------------------------------------------------ + print("\n3. Waiting for key index to become Active...") + active_key = await wait_for_key_active(client, TABLE_NAME, KEY_NAME) + print(f" Key is Active: {active_key.schema_name}") + + # ------------------------------------------------------------------ + # Step 4: Upsert records (creates new) + # ------------------------------------------------------------------ + print("\n4a. Upsert single record (PATCH, creates new)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-001"}, + record={"new_productname": "Widget A", "new_price": 9.99}, + ), + ], + ) + print(" Upserted EXT-001 (single)") + + print("\n4b. Upsert second record (single PATCH)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-002"}, + record={"new_productname": "Widget B", "new_price": 19.99}, + ), + ], + ) + print(" Upserted EXT-002 (single)") + + print("\n4c. Upsert multiple records (UpsertMultiple bulk)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-003"}, + record={"new_productname": "Widget C", "new_price": 29.99}, + ), + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-004"}, + record={"new_productname": "Widget D", "new_price": 39.99}, + ), + ], + ) + print(" Upserted EXT-003, EXT-004 (bulk)") + + # ------------------------------------------------------------------ + # Step 5a: Upsert single update (PATCH, record exists) + # ------------------------------------------------------------------ + print("\n5a. Upsert single record (update existing via PATCH)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-001"}, + record={"new_productname": "Widget A v2", "new_price": 12.99}, + ), + ], + ) + print(" Updated EXT-001 (single)") + + # ------------------------------------------------------------------ + # Step 5b: Upsert multiple update (UpsertMultiple, records exist) + # ------------------------------------------------------------------ + print("\n5b. Upsert multiple records (update existing via UpsertMultiple)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-003"}, + record={"new_productname": "Widget C v2", "new_price": 31.99}, + ), + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-004"}, + record={"new_productname": "Widget D v2", "new_price": 41.99}, + ), + ], + ) + print(" Updated EXT-003, EXT-004 (bulk)") + + # ------------------------------------------------------------------ + # Step 6: Verify + # ------------------------------------------------------------------ + print("\n6. Verifying records...") + async for record in client.records.list_pages( + TABLE_NAME, + select=["new_productname", "new_price", KEY_COLUMN.lower()], + ): + for item in record: + ext_id = item.get(KEY_COLUMN.lower(), "?") + name = item.get("new_productname", "?") + price = item.get("new_price", "?") + print(f" {ext_id}: {name} @ ${price}") + + # ------------------------------------------------------------------ + # Step 7: List alternate keys + # ------------------------------------------------------------------ + print("\n7. Listing alternate keys...") + keys = await client.tables.get_alternate_keys(TABLE_NAME) + for k in keys: + print(f" {k.schema_name}: columns={k.key_attributes}, status={k.status}") + + # ------------------------------------------------------------------ + # Step 8: Cleanup + # ------------------------------------------------------------------ + cleanup = input("\n8. Delete table and cleanup? (Y/n): ").strip() or "y" + if cleanup.lower() in ("y", "yes"): + try: + # Delete alternate key first + for k in keys: + await client.tables.delete_alternate_key(TABLE_NAME, k.metadata_id) + print(f" Deleted key: {k.schema_name}") + await asyncio.sleep(5) + await backoff(lambda: client.tables.delete(TABLE_NAME)) + print(f" Deleted table: {TABLE_NAME}") + except Exception as e: # noqa: BLE001 + print(f" Cleanup error: {e}") + else: + print(" Table kept for inspection.") + finally: + await credential.close() + + print("\nDone.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/batch.py b/examples/aio/advanced/batch.py new file mode 100644 index 00000000..7023a85b --- /dev/null +++ b/examples/aio/advanced/batch.py @@ -0,0 +1,280 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async batch operations example for the Dataverse Python SDK. + +Async equivalent of examples/advanced/batch.py. + +Demonstrates how to use client.batch to send multiple operations in a single +HTTP request to the Dataverse Web API using the async client. + +Requirements: + pip install PowerPlatform-Dataverse-Client azure-identity +""" + +from __future__ import annotations + +import asyncio +import sys + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + +async def main(): + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 1: Record CRUD in a single batch") + + batch = client.batch.new() + + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) + + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) + + result = await batch.execute() + + print( + f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}" + ) + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + async with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = await batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", + ) + + # Read table metadata + batch.tables.get("new_Product") + + # List all non-private tables + batch.tables.list() + + result = await batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = await batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # async with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.retrieve("account", account_id, select=["name", "statecode"]) + # + # result = await batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.retrieve("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = await batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( + [ + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, + ] + ) + batch = client.batch.new() + batch.dataframe.create("account", df) + result = await batch.execute() + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = await batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = await batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: + + batch = client.batch.new() + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") + + result = await batch.execute() + + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue + + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") + + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") + + # Clean up demo records + for rid in result.entity_ids: + await client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + await client.records.delete("account", rid) + finally: + await credential.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/dataframe_operations.py b/examples/aio/advanced/dataframe_operations.py new file mode 100644 index 00000000..463f38fe --- /dev/null +++ b/examples/aio/advanced/dataframe_operations.py @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async DataFrame Operations Walkthrough + +Async equivalent of examples/advanced/dataframe_operations.py. + +This example demonstrates how to use the async pandas DataFrame extension +methods for CRUD operations with Microsoft Dataverse. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys +import uuid + +import pandas as pd +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw + + +async def main(): + # -- Setup & Authentication ------------------------------------ + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + await _run_walkthrough(client) + finally: + await credential.close() + + +async def _run_walkthrough(client): + table = input("Enter table schema name to use [default: account]: ").strip() or "account" + print(f"[INFO] Using table: {table}") + + # Unique tag to isolate test records from existing data + tag = uuid.uuid4().hex[:8] + test_filter = f"contains(name,'{tag}')" + print(f"[INFO] Using tag '{tag}' to identify test records") + + select_cols = ["name", "telephone1", "websiteurl", "lastonholdtime"] + + # -- 1. Create records from a DataFrame ------------------------ + print("\n" + "-" * 60) + print("1. Create records from a DataFrame") + print("-" * 60) + + new_accounts = pd.DataFrame( + [ + { + "name": f"Contoso_{tag}", + "telephone1": "555-0100", + "websiteurl": "https://contoso.com", + "lastonholdtime": pd.Timestamp("2024-06-15 10:30:00"), + }, + {"name": f"Fabrikam_{tag}", "telephone1": "555-0200", "websiteurl": None, "lastonholdtime": None}, + { + "name": f"Northwind_{tag}", + "telephone1": None, + "websiteurl": "https://northwind.com", + "lastonholdtime": pd.Timestamp("2024-12-01 08:00:00"), + }, + ] + ) + print(f" Input DataFrame:\n{new_accounts.to_string(index=False)}\n") + + # create returns a Series of GUIDs aligned with the input rows + new_accounts["accountid"] = await client.dataframe.create(table, new_accounts) + print(f"[OK] Created {len(new_accounts)} records") + print(f" IDs: {new_accounts['accountid'].tolist()}") + + # -- 2. Query records as a DataFrame ------------------------- + print("\n" + "-" * 60) + print("2. Query records as a DataFrame") + print("-" * 60) + + result = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + df_all = result.to_dataframe() + print(f"[OK] Got {len(df_all)} records in one DataFrame") + print(f" Columns: {list(df_all.columns)}") + print(f"{df_all.to_string(index=False)}") + + # -- 3. Limit results with top ------------------------------ + print("\n" + "-" * 60) + print("3. Limit results with top") + print("-" * 60) + + result_top2 = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).top(2).execute() + df_top2 = result_top2.to_dataframe() + print(f"[OK] Got {len(df_top2)} records with top=2") + print(f"{df_top2.to_string(index=False)}") + + # -- 4. Fetch a single record by ID ---------------------------- + print("\n" + "-" * 60) + print("4. Fetch a single record by ID") + print("-" * 60) + + first_id = new_accounts["accountid"].iloc[0] + print(f" Fetching record {first_id}...") + result_single = await client.query.builder(table).select(*select_cols).where(col("accountid") == first_id).execute() + single = result_single.to_dataframe() + print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") + + # -- 5. Update records with different values per row ----------- + print("\n" + "-" * 60) + print("5. Update records with different values per row") + print("-" * 60) + + new_accounts["telephone1"] = ["555-1100", "555-1200", "555-1300"] + print(f" New telephone numbers: {new_accounts['telephone1'].tolist()}") + await client.dataframe.update(table, new_accounts[["accountid", "telephone1"]], id_column="accountid") + print("[OK] Updated 3 records") + + # Verify the updates + result_verified = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + verified = result_verified.to_dataframe() + print(f" Verified:\n{verified.to_string(index=False)}") + + # -- 6. Broadcast update (same value to all records) ----------- + print("\n" + "-" * 60) + print("6. Broadcast update (same value to all records)") + print("-" * 60) + + broadcast_df = new_accounts[["accountid"]].copy() + broadcast_df["websiteurl"] = "https://updated.example.com" + print(f" Setting websiteurl to 'https://updated.example.com' for all {len(broadcast_df)} records") + await client.dataframe.update(table, broadcast_df, id_column="accountid") + print("[OK] Broadcast update complete") + + # Verify all records have the same websiteurl + result_bc = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print(f" Verified:\n{result_bc.to_dataframe().to_string(index=False)}") + + # Default: NaN/None fields are skipped (not overridden on server) + print("\n Updating with NaN values (default: clear_nulls=False, fields should stay unchanged)...") + sparse_df = pd.DataFrame( + [ + {"accountid": new_accounts["accountid"].iloc[0], "telephone1": "555-9999", "websiteurl": None}, + ] + ) + await client.dataframe.update(table, sparse_df, id_column="accountid") + result_sparse = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print( + f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n" + f"{result_sparse.to_dataframe().to_string(index=False)}" + ) + + # Opt-in: clear_nulls=True sends None as null to clear the field + print("\n Clearing websiteurl for Contoso with clear_nulls=True...") + clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) + await client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) + result_clear = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print(f" Verified (Contoso websiteurl should be empty):\n" f"{result_clear.to_dataframe().to_string(index=False)}") + + # -- 7. Delete records by passing a Series of GUIDs ------------ + print("\n" + "-" * 60) + print("7. Delete records by passing a Series of GUIDs") + print("-" * 60) + + print(f" Deleting {len(new_accounts)} records...") + await client.dataframe.delete(table, new_accounts["accountid"], use_bulk_delete=False) + print(f"[OK] Deleted {len(new_accounts)} records") + + # Verify deletions -- filter for our tagged records should return 0 + result_remaining = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + remaining = result_remaining.to_dataframe() + print(f" Verified: {len(remaining)} test records remaining (expected 0)") + + print("\n" + "=" * 60) + print("[OK] Async DataFrame operations walkthrough complete!") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/datascience_risk_assessment.py b/examples/aio/advanced/datascience_risk_assessment.py new file mode 100644 index 00000000..8d88fd68 --- /dev/null +++ b/examples/aio/advanced/datascience_risk_assessment.py @@ -0,0 +1,665 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Data Science Risk Assessment Pipeline + +Async equivalent of examples/advanced/datascience_risk_assessment.py. + +End-to-end example: Extract Dataverse data concurrently into DataFrames, +run statistical analysis, generate LLM-powered risk summaries, and write +results back to Dataverse -- a realistic data analyst / data scientist workflow. + +Pipeline flow: + Dataverse SDK (async) --> Pandas DataFrame --> Analysis + LLM --> Write-back & Reports + +The three Dataverse extraction queries (accounts, cases, opportunities) run +concurrently via asyncio.gather(), reducing wall-clock time for the extract step. + +Scenario: + A financial services company tracks customer accounts, service cases, and + revenue opportunities in Dataverse. The risk team needs to: + 1) Pull data from multiple tables into DataFrames (concurrently) + 2) Compute risk scores using statistical analysis (pandas/numpy) + 3) Classify and summarize risk using an LLM + 4) Write risk assessments back to Dataverse + 5) Produce a summary report + + Note: This example reads from existing Dataverse tables (account, + incident, opportunity) and does not create or delete any tables. + Step 4 (write-back) is disabled by default -- uncomment it in + run_risk_pipeline() to write risk scores back to account records. + +Prerequisites (required -- included in SDK dependencies): + pip install PowerPlatform-Dataverse-Client + pip install azure-identity + +Additional libraries (optional -- used for visualization and LLM): + pip install matplotlib + pip install azure-ai-inference # Option A: Azure AI Foundry / Azure OpenAI + pip install openai # Option B: OpenAI / Azure OpenAI +""" + +import asyncio +import sys +import warnings +from pathlib import Path +from textwrap import dedent + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw + +# -- Optional imports (graceful degradation if not installed) ------ + +try: + import matplotlib + + matplotlib.use("Agg") # non-interactive backend (no GUI required) + import matplotlib.pyplot as plt + + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + + +# ================================================================ +# LLM Provider Configuration +# ================================================================ +# Same providers as the sync version. LLM calls are kept synchronous +# here since they are CPU-light blocking calls. Replace with async +# LLM clients (e.g. openai.AsyncOpenAI) if latency matters. + + +def get_llm_client(provider=None, endpoint=None, api_key=None, model="gpt-4o"): + """Create an LLM client using the specified (or first available) provider. + + Returns a callable: llm_complete(system_prompt, user_prompt) -> str + Returns None if no provider is available. + """ + providers = [provider] if provider else ["azure-ai-inference", "openai", "copilot-sdk"] + for p in providers: + client = _try_init_provider(p, endpoint, api_key, model) + if client is not None: + return client + return None + + +def _wrap_with_logging(raw_complete, provider_name, model_name): + import time + + log = [] + + def complete(system_prompt, user_prompt): + start = time.time() + response = raw_complete(system_prompt, user_prompt) + elapsed = time.time() - start + log.append( + { + "provider": provider_name, + "model": model_name, + "system_prompt": system_prompt, + "user_prompt": user_prompt, + "response": response, + "elapsed_seconds": round(elapsed, 2), + } + ) + return response + + complete.log = log + complete.provider_name = provider_name + complete.model_name = model_name + return complete + + +def _try_init_provider(name, endpoint, api_key, model): + if name == "azure-ai-inference": + return _init_azure_ai(endpoint, api_key, model) + elif name == "openai": + return _init_openai(endpoint, api_key, model) + elif name == "copilot-sdk": + return _init_copilot_sdk() + return None + + +def _init_azure_ai(endpoint, api_key, model): + try: + from azure.ai.inference import ChatCompletionsClient + from azure.ai.inference.models import SystemMessage, UserMessage + from azure.core.credentials import AzureKeyCredential + except ImportError: + return None + + if not endpoint or not api_key: + return None + + client = ChatCompletionsClient(endpoint=endpoint, credential=AzureKeyCredential(api_key)) + + def complete(system_prompt, user_prompt): + response = client.complete( + messages=[SystemMessage(content=system_prompt), UserMessage(content=user_prompt)], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + print("[INFO] LLM provider: Azure AI Inference") + return _wrap_with_logging(complete, "Azure AI Inference", model) + + +def _init_openai(endpoint, api_key, model): + try: + import openai + except ImportError: + return None + + if not api_key: + return None + + if endpoint: + client = openai.AzureOpenAI(azure_endpoint=endpoint, api_key=api_key, api_version="2024-02-01") + else: + client = openai.OpenAI(api_key=api_key) + + def complete(system_prompt, user_prompt): + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + provider_name = "Azure OpenAI" if endpoint else "OpenAI" + print(f"[INFO] LLM provider: {provider_name}") + return _wrap_with_logging(complete, provider_name, model) + + +def _init_copilot_sdk(): + # Uncomment and configure to use your Copilot subscription as the LLM provider. + # from copilot import CopilotClient + # ... + return None + + +# ================================================================ +# Configuration +# ================================================================ + +TABLE_ACCOUNTS = "account" +TABLE_CASES = "incident" +TABLE_OPPORTUNITIES = "opportunity" + +RISK_HIGH = 75 +RISK_MEDIUM = 40 + +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "risk_assessment_output" + + +async def main(): + """Entry point -- authenticate and run the async pipeline.""" + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + await run_risk_pipeline(client) + finally: + await credential.close() + + +# ================================================================ +# Step 1: Extract -- Pull data concurrently with asyncio.gather +# ================================================================ + + +async def step1_extract(client): + """Extract accounts, cases, and opportunities concurrently.""" + print("\n" + "=" * 60) + print("STEP 1: Extract data from Dataverse (concurrently)") + print("=" * 60) + + # All three queries run in parallel -- significant speedup vs sequential. + accounts_result, cases_result, opps_result = await asyncio.gather( + client.query.builder(TABLE_ACCOUNTS) + .select("accountid", "name", "revenue", "numberofemployees", "industrycode") + .where(col("statecode") == 0) + .top(200) + .execute(), + client.query.builder(TABLE_CASES) + .select("incidentid", "_customerid_value", "title", "severitycode", "prioritycode", "createdon") + .where(raw("statecode eq 0")) + .top(1000) + .execute(), + client.query.builder(TABLE_OPPORTUNITIES) + .select( + "opportunityid", + "_parentaccountid_value", + "name", + "estimatedvalue", + "closeprobability", + "estimatedclosedate", + ) + .where(col("statecode") == 0) + .top(1000) + .execute(), + ) + + accounts = accounts_result.to_dataframe() + cases = cases_result.to_dataframe() + opportunities = opps_result.to_dataframe() + + print(f"[OK] Extracted {len(accounts)} active accounts") + print(f"[OK] Extracted {len(cases)} open cases") + print(f"[OK] Extracted {len(opportunities)} active opportunities") + + return accounts, cases, opportunities + + +# ================================================================ +# Step 2: Transform & Analyze -- Statistical risk scoring +# ================================================================ + + +def step2_analyze(accounts, cases, opportunities): + """Compute risk scores using pandas statistical operations (pure Python, unchanged).""" + print("\n" + "=" * 60) + print("STEP 2: Statistical analysis -- compute risk scores") + print("=" * 60) + + if not cases.empty and "_customerid_value" in cases.columns: + case_stats = ( + cases.groupby("_customerid_value") + .agg( + total_cases=("incidentid", "count"), + high_severity_cases=("severitycode", lambda x: (x == 1).sum()), + avg_priority=("prioritycode", "mean"), + ) + .reset_index() + .rename(columns={"_customerid_value": "accountid"}) + ) + else: + case_stats = pd.DataFrame(columns=["accountid", "total_cases", "high_severity_cases", "avg_priority"]) + + if not opportunities.empty and "_parentaccountid_value" in opportunities.columns: + opportunities = opportunities.copy() + opportunities["_weighted_value"] = ( + pd.to_numeric(opportunities["estimatedvalue"], errors="coerce").fillna(0) + * pd.to_numeric(opportunities["closeprobability"], errors="coerce").fillna(0) + / 100 + ) + opp_stats = ( + opportunities.groupby("_parentaccountid_value") + .agg( + total_opportunities=("opportunityid", "count"), + pipeline_value=("estimatedvalue", "sum"), + avg_close_probability=("closeprobability", "mean"), + weighted_pipeline=("_weighted_value", "sum"), + ) + .reset_index() + .rename(columns={"_parentaccountid_value": "accountid"}) + ) + else: + opp_stats = pd.DataFrame( + columns=[ + "accountid", + "total_opportunities", + "pipeline_value", + "avg_close_probability", + "weighted_pipeline", + ] + ) + + risk_df = accounts.merge(case_stats, on="accountid", how="left") + risk_df = risk_df.merge(opp_stats, on="accountid", how="left") + + for c in ["revenue", "numberofemployees"]: + if c in risk_df.columns: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0) + + for c in ["total_cases", "high_severity_cases"]: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0).astype(int) + for c in ["avg_priority", "pipeline_value", "avg_close_probability", "weighted_pipeline"]: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0).astype(float) + risk_df["total_opportunities"] = ( + pd.to_numeric(risk_df["total_opportunities"], errors="coerce").fillna(0).astype(int) + ) + + risk_df["risk_score"] = compute_risk_score(risk_df) + risk_df["risk_tier"] = risk_df["risk_score"].apply(classify_risk) + + print(f"[OK] Computed risk scores for {len(risk_df)} accounts") + print(f" High risk: {(risk_df['risk_tier'] == 'High').sum()}") + print(f" Medium risk: {(risk_df['risk_tier'] == 'Medium').sum()}") + print(f" Low risk: {(risk_df['risk_tier'] == 'Low').sum()}") + + print("\n Risk score distribution:") + print(f" Mean: {risk_df['risk_score'].mean():.1f}") + print(f" Median: {risk_df['risk_score'].median():.1f}") + print(f" Std: {risk_df['risk_score'].std():.1f}") + print(f" Min: {risk_df['risk_score'].min():.1f}") + print(f" Max: {risk_df['risk_score'].max():.1f}") + + return risk_df + + +def compute_risk_score(df): + """Compute a 0-100 risk score from multiple factors.""" + scores = pd.Series(0.0, index=df.index) + + case_total = df["total_cases"].clip(lower=1) + severity_ratio = df["high_severity_cases"] / case_total + scores += severity_ratio * 35 + + if df["total_cases"].max() > 0: + case_pctile = df["total_cases"].rank(pct=True) + scores += case_pctile * 25 + else: + scores += 12.5 + + max_pipeline = df["weighted_pipeline"].max() + if max_pipeline > 0: + pipeline_strength = df["weighted_pipeline"] / max_pipeline + scores += (1 - pipeline_strength) * 20 + else: + scores += 10 + + close_risk = (100 - df["avg_close_probability"]) / 100 + scores += close_risk * 20 + + return scores.clip(0, 100).round(1) + + +def classify_risk(score): + if score >= RISK_HIGH: + return "High" + elif score >= RISK_MEDIUM: + return "Medium" + return "Low" + + +# ================================================================ +# Step 3: LLM Summarization +# ================================================================ + + +def step3_summarize(risk_df, llm_complete=None): + """Generate per-account risk summaries using LLM or template fallback.""" + print("\n" + "=" * 60) + print("STEP 3: Generate risk summaries") + print("=" * 60) + + flagged = risk_df[risk_df["risk_tier"].isin(["High", "Medium"])].copy() + print(f"[INFO] Generating summaries for {len(flagged)} flagged accounts") + + if llm_complete is not None: + summaries = _summarize_with_llm(flagged, llm_complete) + if hasattr(llm_complete, "log") and llm_complete.log: + _export_llm_log(llm_complete) + else: + print("[INFO] No LLM provider configured -- using template-based summarization") + summaries = _summarize_with_template(flagged) + + flagged["risk_summary"] = summaries + summary_map = dict(zip(flagged["accountid"], flagged["risk_summary"])) + risk_df["risk_summary"] = risk_df["accountid"].map(summary_map).fillna("Low risk -- no action needed.") + + print(f"[OK] Generated {len(summaries)} risk summaries") + + top_risk = risk_df.nlargest(3, "risk_score") + for _, row in top_risk.iterrows(): + print(f"\n Account: {row.get('name', 'Unknown')}") + print(f" Risk Score: {row['risk_score']} ({row['risk_tier']})") + print(f" Summary: {row['risk_summary'][:120]}...") + + return risk_df + + +def _summarize_with_llm(flagged_df, llm_complete): + system_prompt = ( + "You are a customer risk analyst at a financial services company. " + "Write exactly 2-3 sentences per account. " + "Sentence 1: State the risk level and primary driver. " + "Sentence 2: Quantify the key metric(s) behind the risk. " + "Sentence 3 (if needed): Recommend one specific action. " + "Use plain business language. Do not use bullet points or markdown." + ) + + summaries = [] + for _, row in flagged_df.iterrows(): + user_prompt = dedent(f"""\ + Summarize the risk for this account: + + Account Name: {row.get("name", "Unknown")} + Risk Score: {row["risk_score"]:.0f}/100 ({row["risk_tier"]} risk) + Open Support Cases: {row["total_cases"]} total, {row["high_severity_cases"]} high-severity + Revenue Pipeline: ${row["pipeline_value"]:,.0f} total, ${row["weighted_pipeline"]:,.0f} probability-weighted + Average Deal Close Probability: {row["avg_close_probability"]:.0f}% + """) + summaries.append(llm_complete(system_prompt, user_prompt)) + + return summaries + + +def _summarize_with_template(flagged_df): + summaries = [] + for _, row in flagged_df.iterrows(): + name = row.get("name", "Unknown") + parts = [] + + if row["high_severity_cases"] > 0: + parts.append(f"{row['high_severity_cases']} high-severity cases require immediate attention") + if row["total_cases"] > 5: + parts.append(f"elevated case volume ({row['total_cases']} open)") + if row["weighted_pipeline"] < 10000: + parts.append("weak revenue pipeline") + if row["avg_close_probability"] < 30: + parts.append(f"low close probability ({row['avg_close_probability']:.0f}%)") + if not parts: + parts.append("multiple moderate risk factors detected") + + summary = ( + f"{name} has a {row['risk_tier'].lower()} risk score of " + f"{row['risk_score']:.0f}/100. Key factors: {'; '.join(parts)}. " + f"Recommend proactive outreach and account review." + ) + summaries.append(summary) + + return summaries + + +def _export_llm_log(llm_complete, include_prompts=False): + log_path = OUTPUT_DIR / "llm_interactions.txt" + with open(log_path, "w", encoding="utf-8") as f: + f.write("LLM Interaction Log\n") + f.write("=" * 70 + "\n") + f.write(f"Provider: {llm_complete.provider_name}\n") + f.write(f"Model: {llm_complete.model_name}\n") + f.write(f"Total calls: {len(llm_complete.log)}\n") + total_time = sum(entry["elapsed_seconds"] for entry in llm_complete.log) + f.write(f"Total time: {total_time:.1f}s\n") + f.write("=" * 70 + "\n\n") + + for i, entry in enumerate(llm_complete.log, 1): + f.write(f"--- Call {i} ({entry['elapsed_seconds']:.2f}s) ---\n\n") + if include_prompts: + f.write(f"[System Prompt]\n{entry['system_prompt']}\n\n") + f.write(f"[User Prompt]\n{entry['user_prompt']}\n\n") + f.write(f"[Response]\n{entry['response']}\n\n") + else: + f.write(f"[Response length: {len(entry['response'])} chars]\n\n") + + print(f"[OK] LLM interaction log saved to {log_path}") + + +# ================================================================ +# Step 4: Write-back +# ================================================================ + + +async def step4_writeback(client, risk_df): + """Write risk scores and summaries back to Dataverse accounts.""" + print("\n" + "=" * 60) + print("STEP 4: Write risk assessments back to Dataverse") + print("=" * 60) + + update_df = risk_df[["accountid", "description"]].copy() + update_df["description"] = risk_df.apply( + lambda r: f"[Risk: {r['risk_tier']} ({r['risk_score']:.0f}/100)] {r['risk_summary']}", + axis=1, + ) + + await client.dataframe.update(TABLE_ACCOUNTS, update_df, id_column="accountid") + print(f"[OK] Updated {len(update_df)} account records with risk assessments") + + +# ================================================================ +# Step 5: Report +# ================================================================ + + +def step5_report(risk_df): + """Generate a summary report with optional visualization.""" + print("\n" + "=" * 60) + print("STEP 5: Risk assessment report") + print("=" * 60) + + tier_summary = ( + risk_df.groupby("risk_tier") + .agg( + count=("accountid", "count"), + avg_score=("risk_score", "mean"), + total_cases=("total_cases", "sum"), + total_pipeline=("pipeline_value", "sum"), + ) + .round(1) + ) + print("\nRisk Tier Summary:") + print(tier_summary.to_string()) + + top10 = risk_df.nlargest(10, "risk_score")[ + ["name", "risk_score", "risk_tier", "total_cases", "high_severity_cases", "pipeline_value"] + ] + print("\nTop 10 Highest Risk Accounts:") + print(top10.to_string(index=False)) + + if HAS_MATPLOTLIB: + _generate_charts(risk_df) + else: + print("\n[INFO] Install matplotlib for risk visualization charts") + + risk_df.to_csv(OUTPUT_DIR / "risk_scores.csv", index=False) + top10.to_csv(OUTPUT_DIR / "top10_risk.csv", index=False) + tier_summary.to_csv(OUTPUT_DIR / "tier_summary.csv") + print(f"\n[OK] Exported CSV reports to {OUTPUT_DIR}/") + + print("\n[OK] Risk assessment pipeline complete!") + + +def _generate_charts(risk_df): + fig, axes = plt.subplots(1, 3, figsize=(16, 5)) + fig.suptitle("Customer Account Risk Assessment", fontsize=14, fontweight="bold") + + axes[0].hist(risk_df["risk_score"], bins=20, color="#4472C4", edgecolor="white") + axes[0].axvline(RISK_HIGH, color="red", linestyle="--", label=f"High ({RISK_HIGH})") + axes[0].axvline(RISK_MEDIUM, color="orange", linestyle="--", label=f"Medium ({RISK_MEDIUM})") + axes[0].set_title("Risk Score Distribution") + axes[0].set_xlabel("Risk Score") + axes[0].set_ylabel("Number of Accounts") + axes[0].legend() + + tier_counts = risk_df["risk_tier"].value_counts() + colors = {"High": "#FF4444", "Medium": "#FFA500", "Low": "#44BB44"} + axes[1].pie( + tier_counts.values, + labels=tier_counts.index, + colors=[colors.get(t, "#888") for t in tier_counts.index], + autopct="%1.0f%%", + startangle=90, + ) + axes[1].set_title("Risk Tier Breakdown") + + axes[2].scatter( + risk_df["total_cases"], + risk_df["pipeline_value"], + c=risk_df["risk_score"], + cmap="RdYlGn_r", + alpha=0.7, + edgecolors="gray", + s=60, + ) + axes[2].set_title("Cases vs Pipeline (color = risk)") + axes[2].set_xlabel("Open Cases") + axes[2].set_ylabel("Pipeline Value ($)") + + plt.tight_layout() + chart_path = OUTPUT_DIR / "risk_assessment_report.png" + plt.savefig(chart_path, dpi=150, bbox_inches="tight") + print(f"[OK] Saved {chart_path}") + + +# ================================================================ +# Pipeline Orchestrator +# ================================================================ + + +async def run_risk_pipeline(client): + """Run the full async risk assessment pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + print("\n" + "#" * 60) + print(" ASYNC CUSTOMER RISK ASSESSMENT PIPELINE") + print(" Dataverse SDK (async) -> Pandas -> Analysis -> LLM -> Write-back") + print("#" * 60) + + # Step 1: Extract data concurrently + accounts, cases, opportunities = await step1_extract(client) + + if accounts.empty: + print("[WARN] No accounts found -- nothing to analyze.") + return + + # Step 2: Statistical analysis (pure Python -- synchronous) + risk_df = step2_analyze(accounts, cases, opportunities) + + # Step 3: LLM-powered risk summarization (synchronous LLM calls) + # Configure your LLM provider (uncomment one): + # Option A: Azure AI Inference + # llm = get_llm_client("azure-ai-inference", endpoint="https://...", api_key="...") + # Option B: OpenAI + # llm = get_llm_client("openai", api_key="sk-...") + # Option C: Azure OpenAI (via openai package) + # llm = get_llm_client("openai", endpoint="https://...", api_key="...") + llm = None # Set to get_llm_client(...) to enable LLM summarization + risk_df = step3_summarize(risk_df, llm_complete=llm) + + # Step 4: Write results back to Dataverse (async) + # Uncomment the next line to write back (requires custom columns on account table) + # await step4_writeback(client, risk_df) + print("\n[INFO] Step 4 (write-back) is commented out by default.") + print(" Uncomment step4_writeback() after adding custom columns to account table.") + + # Step 5: Generate summary report + charts (synchronous) + step5_report(risk_df) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/fetchxml.py b/examples/aio/advanced/fetchxml.py new file mode 100644 index 00000000..95fd8811 --- /dev/null +++ b/examples/aio/advanced/fetchxml.py @@ -0,0 +1,574 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async end-to-end FetchXML examples for Dataverse. + +Async equivalent of examples/advanced/fetchxml.py. + +Demonstrates ``await client.query.fetchxml().execute()`` and +``async for page in client.query.fetchxml().execute_pages()`` across +scenarios where FetchXML is required or preferred over OData/SQL: + +- Basic attribute queries +- operators (eq, like, in, null, not-null, between) +- (inner and outer joins) +- Ordering +- Page-size control with automatic paging-cookie propagation +- Aggregate queries (count, sum, avg, min, max, group-by) +- Built-in system tables (account → contact join) + +FetchXML is the right tool when: +- You need a JOIN type OData $expand cannot express (many-to-many, outer link) +- You need server-side aggregates (count, sum, avg) without GROUP BY SQL +- You need ```` operators unavailable in OData ($filter) + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import asyncio +import sys + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK -- Async FetchXML End-to-End Examples") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_examples(client) + finally: + await credential.close() + + +async def _run_examples(client): + project_table = "new_FXDemoProject" + task_table = "new_FXDemoTask" + + # =================================================================== + # 2. Create tables and seed data + # =================================================================== + heading(2, "Create Tables & Seed Data") + + log_call(f"await client.tables.get('{project_table}')") + if await client.tables.get(project_table): + print(f"[OK] Table already exists: {project_table}") + else: + log_call(f"await client.tables.create('{project_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + project_table, + { + "new_Code": "string", + "new_Budget": "decimal", + "new_Active": "bool", + "new_Region": "int", + }, + ) + ) + print(f"[OK] Created table: {project_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {project_table} (skipped)") + else: + raise + + log_call(f"await client.tables.get('{task_table}')") + if await client.tables.get(task_table): + print(f"[OK] Table already exists: {task_table}") + else: + log_call(f"await client.tables.create('{task_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + task_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {task_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {task_table} (skipped)") + else: + raise + + print("\n[INFO] Creating lookup field: tasks → projects ...") + try: + await client.tables.create_lookup_field( + referencing_table=task_table, + lookup_field_name="new_ProjectId", + referenced_table=project_table, + display_name="Project", + ) + print("[OK] Created lookup: new_ProjectId on tasks → projects") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + # Resolve entity set name for @odata.bind + project_set = f"{project_table.lower()}s" + try: + tinfo = await client.tables.get(project_table) + if tinfo: + project_set = tinfo.get("entity_set_name", project_set) + except Exception: + pass + + log_call(f"await client.records.create('{project_table}', [...])") + projects = [ + {"new_Code": "ALPHA", "new_Budget": 50000, "new_Active": True, "new_Region": 1}, + {"new_Code": "BRAVO", "new_Budget": 75000, "new_Active": True, "new_Region": 2}, + {"new_Code": "CHARLIE", "new_Budget": 30000, "new_Active": False, "new_Region": 3}, + {"new_Code": "DELTA", "new_Budget": 90000, "new_Active": True, "new_Region": 1}, + {"new_Code": "ECHO", "new_Budget": 42000, "new_Active": True, "new_Region": 2}, + ] + project_ids = await backoff(lambda: client.records.create(project_table, projects)) + print(f"[OK] Seeded {len(project_ids)} projects") + + log_call(f"await client.records.create('{task_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[4]})", + }, + ] + task_ids = await backoff(lambda: client.records.create(task_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks") + + project_logical = project_table.lower() + task_logical = task_table.lower() + project_pk = f"{project_logical}id" + lookup_attr = "new_projectid" + + try: + # =================================================================== + # 3. Basic attribute query + # =================================================================== + heading(3, "Basic Attribute Query") + xml = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(basic attribute query).execute()") + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} projects:") + for r in result: + print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + if result: + print(f" First by index: {result[0].get('new_code')}") + print(f" First by .first(): {result.first().get('new_code')}") + + # =================================================================== + # 4. operators + # =================================================================== + heading(4, " Operators") + + # eq + xml = f""" + + + + + + + """ + log_call('operator="eq" value="ALPHA"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] eq: {[x.get('new_code') for x in r]}") + + # like + xml = f""" + + + + + + + """ + log_call('operator="like" value="%test%"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") + + # in + xml = f""" + + + + + + ALPHA + DELTA + + + + + """ + log_call('operator="in" values=[ALPHA, DELTA]') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] in: {[x.get('new_code') for x in r]}") + + # not-null + xml = f""" + + + + + + + """ + log_call('operator="not-null"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] not-null: {len(r)} tasks have priority set") + + # between + xml = f""" + + + + + + + 40000 + 80000 + + + + + """ + log_call('operator="between" 40000 and 80000') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") + + # =================================================================== + # 5. — inner join + # =================================================================== + heading(5, " Inner Join (Tasks → Projects)") + xml = f""" + + + + + + + + + + + """ + log_call("await client.query.fetchxml(link-entity inner join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows:") + for r in result: + print( + f" Task={r.get('new_title', ''):<25s} " + f"Hours={r.get('new_hours')} " + f"Project={r.get('p.new_code', '')} " + f"Budget={r.get('p.new_budget')}" + ) + except Exception as e: + print(f"[WARN] link-entity join failed: {e}") + + # =================================================================== + # 6. — outer join + # =================================================================== + heading(6, " Outer Join (Projects With or Without Tasks)") + xml = f""" + + + + + + + + + """ + log_call("await client.query.fetchxml(link-entity outer join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows (includes projects with no tasks):") + for r in result[:8]: + print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") + except Exception as e: + print(f"[WARN] outer join failed: {e}") + + # =================================================================== + # 7. Ordering + # =================================================================== + heading(7, "Ordering ( element)") + xml = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(order by hours DESC).execute()") + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Tasks by hours DESC:") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + # =================================================================== + # 8. Paging-cookie propagation + # =================================================================== + heading(8, "Paging-Cookie Propagation") + print( + "[INFO] 'count' sets the page size in FetchXML.\n" + "With count='2' and 7 seeded tasks the server returns pages of 2, 2, 2, 1.\n" + ".execute() collects all pages eagerly; .execute_pages() yields one QueryResult per HTTP page." + ) + xml_paged = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(xml).execute() — eager, all pages collected") + result = await backoff(lambda: client.query.fetchxml(xml_paged).execute()) + print(f"[OK] execute(): {len(result)} total tasks (seeded {len(task_ids)}):") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + log_call("async for page in client.query.fetchxml(xml).execute_pages() — lazy, one QueryResult per page") + page_num = 0 + page_record_count = 0 + async for page in client.query.fetchxml(xml_paged).execute_pages(): + page_num += 1 + page_record_count += len(page) + print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") + print(f"[OK] execute_pages(): {page_record_count} total tasks across {page_num} page(s)") + + # =================================================================== + # 9. Aggregates + # =================================================================== + heading(9, "Aggregate Queries ()") + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(aggregate: count, sum, avg, min, max).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + if result: + row = result.first() + print( + f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " + f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" + ) + except Exception as e: + print(f"[WARN] aggregate failed: {e}") + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(aggregate group-by project).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Hours per project ({len(result)} groups):") + for r in result: + print( + f" {r.get('project_code', ''):<10s} " + f"Tasks={r.get('task_count')} " + f"Hours={r.get('total_hours')}" + ) + except Exception as e: + print(f"[WARN] group-by aggregate failed: {e}") + + # =================================================================== + # 10. Built-in system tables + # =================================================================== + heading(10, "Built-In System Tables (account → contact Join)") + xml = """ + + + + + + + + + """ + log_call("await client.query.fetchxml(account → contact inner join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} account-contact pairs:") + for r in result: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") + except Exception as e: + print(f"[INFO] No account-contact data in this org: {e}") + + finally: + heading(11, "Cleanup") + for tbl in [task_table, project_table]: + log_call(f"await client.tables.delete('{tbl}')") + try: + await backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + if "404" in str(ex) or (isinstance(ex, MetadataError) and "not found" in str(ex).lower()): + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("Async FetchXML Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/file_upload.py b/examples/aio/advanced/file_upload.py new file mode 100644 index 00000000..e44f6a7c --- /dev/null +++ b/examples/aio/advanced/file_upload.py @@ -0,0 +1,361 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async File Upload Example + +Async equivalent of examples/advanced/file_upload.py. + +This example demonstrates file upload capabilities using the async +PowerPlatform-Dataverse-Client SDK with automatic chunking for large files. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import hashlib +import sys +import traceback +from pathlib import Path + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +ATTRIBUTE_VISIBILITY_DELAYS = (0, 3, 10, 20, 35, 50, 70, 90, 120) + +# --- Helpers --- + +_FILE_HASH_CACHE: dict = {} + + +def file_sha256(path: Path): + """Return (hex_digest, size_bytes) for the file, with caching.""" + try: + cached = _FILE_HASH_CACHE.get(path) + if cached: + return cached + h = hashlib.sha256() + size = 0 + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + size += len(chunk) + h.update(chunk) + result = (h.hexdigest(), size) + _FILE_HASH_CACHE[path] = result + return result + except Exception: # noqa: BLE001 + return None, None + + +def generate_test_file(size_mb: int = 10) -> Path: + """Generate a dummy text file of the specified size for testing.""" + test_file = Path(__file__).resolve().parent / f"test_dummy_{size_mb}mb.txt" + target_size = size_mb * 1024 * 1024 + + line = b"The quick brown fox jumps over the lazy dog. " * 2 + b"\n" + with test_file.open("wb") as f: + written = 0 + while written < target_size: + chunk = line * min(1000, (target_size - written) // len(line) + 1) + chunk = chunk[: target_size - written] + f.write(chunk) + written += len(chunk) + + print({"test_file_generated": str(test_file), "size_mb": test_file.stat().st_size / (1024 * 1024)}) + return test_file + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an async operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" [INFO] Backoff succeeded after {retry_count} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); waited {total_delay}s total.") + raise last + + +# --- Table ensure --- +TABLE_SCHEMA_NAME = "new_FileSample" + + +async def ensure_table(client) -> dict: + """Get or create the demo table.""" + existing = await backoff(lambda: client.tables.get(TABLE_SCHEMA_NAME)) + if existing: + print({"table": TABLE_SCHEMA_NAME, "existed": True}) + return existing + info = await backoff(lambda: client.tables.create(TABLE_SCHEMA_NAME, {"new_Title": "string"})) + print({"table": TABLE_SCHEMA_NAME, "existed": False, "metadata_id": info.get("metadata_id")}) + return info + + +# --- Main --- + + +async def main(): + entered = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not entered: + print("No URL entered; exiting.") + sys.exit(1) + base_url = entered.rstrip("/") + + # Mode selection (numeric): + # 1 = small (single PATCH <128MB) + # 2 = chunk (streaming for any size) + # 3 = all (small + chunk) + mode_raw = input("Choose mode: 1) small 2) chunk 3) all [default 3]: ").strip() + if not mode_raw: + mode_raw = "3" + if mode_raw not in {"1", "2", "3"}: + print({"invalid_mode": mode_raw, "fallback": 3}) + mode_raw = "3" + mode_int = int(mode_raw) + run_small = mode_int in (1, 3) + run_chunk = mode_int in (2, 3) + + delete_record_choice = input("Delete the created record at end? (Y/n): ").strip() or "y" + cleanup_record = delete_record_choice.lower() in ("y", "yes", "true", "1") + + delete_table_choice = input("Delete the table at end? (y/N): ").strip() or "n" + cleanup_table = delete_table_choice.lower() in ("y", "yes", "true", "1") + + credential = AsyncInteractiveBrowserCredential() + + # Generate test files before entering the async context + generated_10mb = generate_test_file(10) + generated_8mb = generate_test_file(8) + + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + + # --------------------------- Table ensure --------------------------- + try: + table_info = await ensure_table(client) + except Exception: # noqa: BLE001 + print("Table ensure failed:") + traceback.print_exc() + sys.exit(1) + + entity_set = table_info.get("entity_set_name") + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + name_attr = f"{attr_prefix}_name" + small_file_attr_schema = f"{attr_prefix}_SmallDocument" + chunk_file_attr_schema = f"{attr_prefix}_ChunkDocument" + + # --------------------------- Record create --------------------------- + record_id = None + try: + payload = {name_attr: "Async File Sample Record"} + print({"call": f"client.records.create('{table_schema_name}', payload)"}) + record_id = await backoff(lambda: client.records.create(table_schema_name, payload)) + print({"record_created": True, "id": record_id, "table schema name": table_schema_name}) + except Exception as e: # noqa: BLE001 + print({"record_created": False, "error": str(e)}) + sys.exit(1) + + if not record_id: + print("No record id; aborting upload.") + sys.exit(1) + + # --------------------------- Small single-request upload --------------------------- + if run_small: + print("Small single-request upload demo:") + try: + src_hash, small_file_size = file_sha256(generated_10mb) + + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=small_file_attr_schema, + path=str(generated_10mb), + mode="small", + ) + ) + print({"small_upload_completed": True, "small_source_size": small_file_size}) + + # Download and verify via internal OData client + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{small_file_attr_schema.lower()}/$value" + resp = await od._request("get", dl_url) + content = await resp.read() if hasattr(resp, "read") else (resp.content or b"") + + downloaded_hash = hashlib.sha256(content).hexdigest() if content else None + hash_match = (downloaded_hash == src_hash) if (downloaded_hash and src_hash) else None + print( + { + "small_file_source_size": small_file_size, + "small_file_download_size": len(content), + "small_file_size_match": len(content) == small_file_size, + "small_file_source_sha256_prefix": src_hash[:16] if src_hash else None, + "small_file_download_sha256_prefix": downloaded_hash[:16] if downloaded_hash else None, + "small_file_hash_match": hash_match, + } + ) + + # Replace with 8MB file + print("Small single-request upload demo - REPLACE with 8MB file:") + replace_hash, replace_size = file_sha256(generated_8mb) + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=small_file_attr_schema, + path=str(generated_8mb), + mode="small", + if_none_match=False, + ) + ) + print({"small_replace_upload_completed": True, "small_replace_source_size": replace_size}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{small_file_attr_schema.lower()}/$value" + resp_r = await od._request("get", dl_url) + content_r = await resp_r.read() if hasattr(resp_r, "read") else (resp_r.content or b"") + + dl_hash_r = hashlib.sha256(content_r).hexdigest() if content_r else None + hash_match_r = (dl_hash_r == replace_hash) if (dl_hash_r and replace_hash) else None + print( + { + "small_replace_source_size": replace_size, + "small_replace_download_size": len(content_r), + "small_replace_size_match": len(content_r) == replace_size, + "small_replace_source_sha256_prefix": replace_hash[:16] if replace_hash else None, + "small_replace_download_sha256_prefix": dl_hash_r[:16] if dl_hash_r else None, + "small_replace_hash_match": hash_match_r, + } + ) + except Exception as ex: # noqa: BLE001 + print({"single_upload_failed": str(ex)}) + + # --------------------------- Chunk (streaming) upload --------------------------- + if run_chunk: + print("Streaming chunk upload demo (mode='chunk'):") + try: + src_hash_chunk, src_size_chunk = file_sha256(generated_10mb) + + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=chunk_file_attr_schema, + path=str(generated_10mb), + mode="chunk", + ) + ) + print({"chunk_upload_completed": True}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{chunk_file_attr_schema.lower()}/$value" + resp = await od._request("get", dl_url) + content_chunk = await resp.read() if hasattr(resp, "read") else (resp.content or b"") + + dst_hash_chunk = hashlib.sha256(content_chunk).hexdigest() if content_chunk else None + hash_match_chunk = ( + (dst_hash_chunk == src_hash_chunk) if (dst_hash_chunk and src_hash_chunk) else None + ) + print( + { + "chunk_source_size": src_size_chunk, + "chunk_download_size": len(content_chunk), + "chunk_size_match": len(content_chunk) == src_size_chunk, + "chunk_source_sha256_prefix": src_hash_chunk[:16] if src_hash_chunk else None, + "chunk_download_sha256_prefix": dst_hash_chunk[:16] if dst_hash_chunk else None, + "chunk_hash_match": hash_match_chunk, + } + ) + + # Replace with 8MB file + print("Streaming chunk upload demo - REPLACE with 8MB file:") + replace_hash_c, replace_size_c = file_sha256(generated_8mb) + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=chunk_file_attr_schema, + path=str(generated_8mb), + mode="chunk", + if_none_match=False, + ) + ) + print({"chunk_replace_upload_completed": True}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{chunk_file_attr_schema.lower()}/$value" + resp_rc = await od._request("get", dl_url) + content_rc = await resp_rc.read() if hasattr(resp_rc, "read") else (resp_rc.content or b"") + + dl_hash_rc = hashlib.sha256(content_rc).hexdigest() if content_rc else None + hash_match_rc = (dl_hash_rc == replace_hash_c) if (dl_hash_rc and replace_hash_c) else None + print( + { + "chunk_replace_source_size": replace_size_c, + "chunk_replace_download_size": len(content_rc), + "chunk_replace_size_match": len(content_rc) == replace_size_c, + "chunk_replace_source_sha256_prefix": replace_hash_c[:16] if replace_hash_c else None, + "chunk_replace_download_sha256_prefix": dl_hash_rc[:16] if dl_hash_rc else None, + "chunk_replace_hash_match": hash_match_rc, + } + ) + except Exception as ex: # noqa: BLE001 + print({"chunk_upload_failed": str(ex)}) + + # --------------------------- Cleanup --------------------------- + if cleanup_record and record_id: + try: + print({"call": f"client.records.delete('{table_schema_name}', '{record_id}')"}) + await backoff(lambda: client.records.delete(table_schema_name, record_id)) + print({"record_deleted": True}) + except Exception as e: # noqa: BLE001 + print({"record_deleted": False, "error": str(e)}) + else: + print({"record_deleted": False, "reason": "user opted to keep"}) + + if cleanup_table: + try: + print({"call": f"client.tables.delete('{TABLE_SCHEMA_NAME}')"}) + await backoff(lambda: client.tables.delete(TABLE_SCHEMA_NAME)) + print({"table_deleted": True}) + except Exception as e: # noqa: BLE001 + print({"table_deleted": False, "error": str(e)}) + else: + print({"table_deleted": False, "reason": "user opted to keep"}) + finally: + await credential.close() + + # Clean up generated test files + for f in [generated_10mb, generated_8mb]: + if f and f.exists(): + try: + f.unlink() + print({"test_file_deleted": True, "path": str(f)}) + except Exception as e: # noqa: BLE001 + print({"test_file_deleted": False, "error": str(e)}) + + print("Done.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/prodev_quick_start.py b/examples/aio/advanced/prodev_quick_start.py new file mode 100644 index 00000000..4cc953a8 --- /dev/null +++ b/examples/aio/advanced/prodev_quick_start.py @@ -0,0 +1,475 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Pro-Dev Quick Start + +Async equivalent of examples/advanced/prodev_quick_start.py. + +A developer-focused example that demonstrates the full async SDK lifecycle: +install, authenticate, create a system with 4 related tables, populate +data, query it, and clean up -- all in a single script. + +What this example covers: + 1) SDK installation and authentication + 2) Create 4 custom tables concurrently with asyncio.gather() + 3) Create columns and relationships between tables + 4) Populate with sample data using async DataFrame CRUD + 5) Query and join data across tables + 6) Clean up (delete tables) + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys +import uuid +import warnings +from pathlib import Path + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col + +# -- Table schema names -- +SUFFIX = uuid.uuid4().hex[:6] +TABLE_CUSTOMER = f"new_DemoCustomer{SUFFIX}" +TABLE_PROJECT = f"new_DemoProject{SUFFIX}" +TABLE_TASK = f"new_DemoTask{SUFFIX}" +TABLE_TIMEENTRY = f"new_DemoTimeEntry{SUFFIX}" + +# -- Output folder for exported data -- +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "prodev_output" + + +async def main(): + """Entry point.""" + print("=" * 60) + print(" DATAVERSE PYTHON SDK -- ASYNC PRO-DEV QUICK START") + print("=" * 60) + print() + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser (Azure Identity)...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + try: + await run_demo(client) + except Exception as e: + print(f"\n[ERR] {e}") + print("[INFO] Attempting cleanup...") + await cleanup(client) + raise + finally: + await credential.close() + + +async def run_demo(client): + """Run the full async pro-dev demo pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + # -- Step 1: Create 4 tables concurrently -- + primary_name_col, primary_id_col = await step1_create_tables(client) + + # -- Step 2: Create relationships -- + await step2_create_relationships(client) + + # -- Step 3: Populate with sample data -- + customer_ids, project_ids, task_ids = await step3_populate_data(client, primary_name_col) + + # -- Step 4: Query and analyze -- + await step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col) + + # -- Step 5: Update and delete -- + await step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) + + # -- Step 6: Cleanup -- + do_cleanup = input("\n6. Delete demo tables and cleanup? (Y/n): ").strip() or "y" + if do_cleanup.lower() in ("y", "yes"): + await cleanup(client) + else: + print("[INFO] Tables kept for inspection.") + + print("\n" + "=" * 60) + print("[OK] Async pro-dev quick start demo complete!") + print("=" * 60) + + +# ================================================================ +# Step 1: Create tables (concurrently with asyncio.gather) +# ================================================================ + + +async def step1_create_tables(client): + """Create 4 custom tables sequentially. + + Note: Dataverse holds a metadata customization lock for the duration of + each table-creation request. Concurrent creates (asyncio.gather) trigger + a CustomizationLockException on the server, so tables must be created one + at a time. + """ + print("\n" + "-" * 60) + print("STEP 1: Create 4 custom tables (sequentially)") + print("-" * 60) + + customer_result = await client.tables.create( + TABLE_CUSTOMER, + { + f"{TABLE_CUSTOMER}_Email": "string", + f"{TABLE_CUSTOMER}_Industry": "string", + f"{TABLE_CUSTOMER}_Revenue": "money", + }, + ) + await client.tables.create( + TABLE_PROJECT, + { + f"{TABLE_PROJECT}_Budget": "money", + f"{TABLE_PROJECT}_Status": "string", + f"{TABLE_PROJECT}_StartDate": "datetime", + }, + ) + await client.tables.create( + TABLE_TASK, + { + f"{TABLE_TASK}_Priority": "integer", + f"{TABLE_TASK}_Status": "string", + f"{TABLE_TASK}_EstimatedHours": "decimal", + }, + ) + await client.tables.create( + TABLE_TIMEENTRY, + { + f"{TABLE_TIMEENTRY}_Hours": "decimal", + f"{TABLE_TIMEENTRY}_Date": "datetime", + f"{TABLE_TIMEENTRY}_Description": "string", + }, + ) + + primary_name_col = customer_result.primary_name_attribute + primary_id_col = customer_result.primary_id_attribute + print(f"[OK] Created table: {TABLE_CUSTOMER} (name: {primary_name_col}, id: {primary_id_col})") + print(f"[OK] Created table: {TABLE_PROJECT}") + print(f"[OK] Created table: {TABLE_TASK}") + print(f"[OK] Created table: {TABLE_TIMEENTRY}") + print(f"[OK] All 4 tables created (suffix: {SUFFIX})") + + return primary_name_col, primary_id_col + + +# ================================================================ +# Step 2: Create relationships +# ================================================================ + + +async def step2_create_relationships(client): + """Create relationships between the 4 tables using lookup fields.""" + print("\n" + "-" * 60) + print("STEP 2: Create relationships (lookup fields)") + print("-" * 60) + + # Relationships must be created sequentially -- Dataverse rejects + # concurrent metadata writes to related tables. + await client.tables.create_lookup_field( + referencing_table=TABLE_PROJECT.lower(), + lookup_field_name=f"{TABLE_PROJECT}_CustomerId", + referenced_table=TABLE_CUSTOMER.lower(), + display_name="Customer", + ) + print(f"[OK] {TABLE_CUSTOMER} 1:N {TABLE_PROJECT}") + + await client.tables.create_lookup_field( + referencing_table=TABLE_TASK.lower(), + lookup_field_name=f"{TABLE_TASK}_ProjectId", + referenced_table=TABLE_PROJECT.lower(), + display_name="Project", + ) + print(f"[OK] {TABLE_PROJECT} 1:N {TABLE_TASK}") + + await client.tables.create_lookup_field( + referencing_table=TABLE_TIMEENTRY.lower(), + lookup_field_name=f"{TABLE_TIMEENTRY}_TaskId", + referenced_table=TABLE_TASK.lower(), + display_name="Task", + ) + print(f"[OK] {TABLE_TASK} 1:N {TABLE_TIMEENTRY}") + print("[OK] 3 lookup relationships created (Customer -> Project -> Task -> TimeEntry)") + + +# ================================================================ +# Step 3: Populate with sample data +# ================================================================ + + +async def step3_populate_data(client, primary_name_col): + """Create sample records using client.dataframe.create().""" + print("\n" + "-" * 60) + print("STEP 3: Populate with sample data (async DataFrame CRUD)") + print("-" * 60) + + # -- Customers -- + name_col = primary_name_col + customers_df = pd.DataFrame( + [ + { + name_col: "Contoso Ltd", + f"{TABLE_CUSTOMER}_Email": "info@contoso.com", + f"{TABLE_CUSTOMER}_Industry": "Technology", + f"{TABLE_CUSTOMER}_Revenue": 5000000, + }, + { + name_col: "Fabrikam Inc", + f"{TABLE_CUSTOMER}_Email": "contact@fabrikam.com", + f"{TABLE_CUSTOMER}_Industry": "Manufacturing", + f"{TABLE_CUSTOMER}_Revenue": 12000000, + }, + { + name_col: "Northwind Traders", + f"{TABLE_CUSTOMER}_Email": "sales@northwind.com", + f"{TABLE_CUSTOMER}_Industry": "Retail", + f"{TABLE_CUSTOMER}_Revenue": 3000000, + }, + ] + ) + customer_ids = await client.dataframe.create(TABLE_CUSTOMER, customers_df) + customers_df["id"] = customer_ids + print(f"[OK] Created {len(customers_df)} customers") + + # -- Projects (linked to customers via lookup) -- + customer_lookup = f"{TABLE_PROJECT}_CustomerId@odata.bind" + customer_info = await client.tables.get(TABLE_CUSTOMER) + customer_set = customer_info.get("entity_set_name") if customer_info else TABLE_CUSTOMER.lower() + "s" + projects_df = pd.DataFrame( + [ + { + name_col: "Cloud Migration", + f"{TABLE_PROJECT}_Budget": 250000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-15"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + { + name_col: "ERP Upgrade", + f"{TABLE_PROJECT}_Budget": 500000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-02-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[1]})", + }, + { + name_col: "POS Modernization", + f"{TABLE_PROJECT}_Budget": 150000, + f"{TABLE_PROJECT}_Status": "Planning", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-03-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[2]})", + }, + { + name_col: "Data Analytics Platform", + f"{TABLE_PROJECT}_Budget": 180000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-20"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + ] + ) + project_ids = await client.dataframe.create(TABLE_PROJECT, projects_df) + projects_df["id"] = project_ids + print(f"[OK] Created {len(projects_df)} projects across 3 customers") + + # -- Tasks (linked to projects) -- + task_names = [ + ("Infrastructure Setup", 1, "In Progress", 40), + ("Data Assessment", 2, "Not Started", 20), + ("Testing & QA", 1, "Not Started", 60), + ("Requirements Gathering", 1, "Complete", 30), + ("Development Sprint 1", 1, "In Progress", 80), + ("User Training", 3, "Not Started", 16), + ] + project_assignment = [0, 0, 0, 1, 1, 2] + + project_info = await client.tables.get(TABLE_PROJECT) + project_set = project_info.get("entity_set_name") if project_info else TABLE_PROJECT.lower() + "s" + project_lookup = f"{TABLE_TASK}_ProjectId@odata.bind" + + tasks_data = [ + { + name_col: task_name, + f"{TABLE_TASK}_Priority": priority, + f"{TABLE_TASK}_Status": status, + f"{TABLE_TASK}_EstimatedHours": hours, + project_lookup: f"/{project_set}({project_ids.iloc[project_assignment[i]]})", + } + for i, (task_name, priority, status, hours) in enumerate(task_names) + ] + + tasks_df = pd.DataFrame(tasks_data) + task_ids = await client.dataframe.create(TABLE_TASK, tasks_df) + tasks_df["id"] = task_ids + print(f"[OK] Created {len(tasks_df)} tasks across 4 projects") + + print( + f"\n Total records: {len(customers_df) + len(projects_df) + len(tasks_df)} " + f"({len(customers_df)} customers, {len(projects_df)} projects, {len(tasks_df)} tasks)" + ) + + return customer_ids, project_ids, task_ids + + +# ================================================================ +# Step 4: Query and analyze data +# ================================================================ + + +async def step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col): + """Query data and demonstrate DataFrame analysis.""" + print("\n" + "-" * 60) + print("STEP 4: Query and analyze data") + print("-" * 60) + + name_attr = primary_name_col + + # Query projects and tasks concurrently + project_result, task_result = await asyncio.gather( + client.query.builder(TABLE_PROJECT) + .select(name_attr, f"{TABLE_PROJECT}_Budget", f"{TABLE_PROJECT}_Status") + .execute(), + client.query.builder(TABLE_TASK) + .select(name_attr, f"{TABLE_TASK}_Priority", f"{TABLE_TASK}_Status", f"{TABLE_TASK}_EstimatedHours") + .execute(), + ) + + projects = project_result.to_dataframe() + tasks = task_result.to_dataframe() + + print(f"\n All projects ({len(projects)} rows):") + print(f"{projects.to_string(index=False)}") + + print(f"\n All tasks ({len(tasks)} rows):") + print(f"{tasks.to_string(index=False)}") + + # -- DataFrame analysis -- + hours_col = f"{TABLE_TASK}_EstimatedHours" + status_col = f"{TABLE_TASK}_Status" + budget_col = f"{TABLE_PROJECT}_Budget" + + if hours_col in tasks.columns: + print(f"\n Task hours summary:") + print(f" Total estimated hours: {tasks[hours_col].sum():.0f}") + print(f" Average per task: {tasks[hours_col].mean():.1f}") + print(f" Max single task: {tasks[hours_col].max():.0f}") + + if status_col in tasks.columns: + print(f"\n Tasks by status:") + for status, count in tasks[status_col].value_counts().items(): + print(f" {status}: {count}") + + if budget_col in projects.columns: + print(f"\n Project budget summary:") + print(f" Total budget: ${projects[budget_col].sum():,.0f}") + print(f" Average budget: ${projects[budget_col].mean():,.0f}") + + # Fetch single customer record by ID + first_id = customer_ids.iloc[0] + single_result = await client.query.builder(TABLE_CUSTOMER).where(col(primary_id_col) == first_id).execute() + single = single_result.to_dataframe() + print(f"\n Single customer record (by ID):") + print(f"{single.to_string(index=False)}") + + # -- Export query results to CSV -- + projects.to_csv(OUTPUT_DIR / "projects.csv", index=False) + tasks.to_csv(OUTPUT_DIR / "tasks.csv", index=False) + single.to_csv(OUTPUT_DIR / "single_customer.csv", index=False) + print(f"\n[OK] Exported query results to {OUTPUT_DIR}/") + + +# ================================================================ +# Step 5: Update and delete records +# ================================================================ + + +async def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): + """Demonstrate update and delete with DataFrames.""" + print("\n" + "-" * 60) + print("STEP 5: Update and delete records") + print("-" * 60) + + status_col = f"{TABLE_TASK}_Status" + + # Update: mark first two tasks as "Complete" + update_df = pd.DataFrame( + { + primary_id_col: [task_ids.iloc[0], task_ids.iloc[1]], + status_col: ["Complete", "Complete"], + } + ) + await client.dataframe.update(TABLE_TASK, update_df, id_column=primary_id_col) + print(f"[OK] Updated 2 tasks to 'Complete'") + + # Delete: remove the last task + delete_ids = pd.Series([task_ids.iloc[-1]]) + await client.dataframe.delete(TABLE_TASK, delete_ids) + print(f"[OK] Deleted 1 task") + + # Verify + result = await client.query.builder(TABLE_TASK).select(primary_name_col, status_col).execute() + remaining = result.to_dataframe() + print(f"\n Remaining tasks ({len(remaining)}):") + print(f"{remaining.to_string(index=False)}") + + +# ================================================================ +# Cleanup +# ================================================================ + + +async def cleanup(client): + """Delete all demo tables. + + Tables must be deleted leaf-to-root (TimeEntry → Task → Project → Customer) + because each table holds a lookup field referencing the next. Dataverse may + also return transient SQL-deadlock errors during metadata operations, so we + retry failed deletions until all tables are gone or no further progress is + made. + """ + print("\n" + "-" * 60) + print("CLEANUP: Removing demo tables") + print("-" * 60) + + remaining = [TABLE_TIMEENTRY, TABLE_TASK, TABLE_PROJECT, TABLE_CUSTOMER] + while remaining: + failed = [] + for table in remaining: + try: + await client.tables.delete(table) + print(f"[OK] Deleted table: {table}") + except Exception as e: + failed.append((table, e)) + + if len(failed) == len(remaining): + # No progress — report and stop to avoid an infinite loop. + for table, e in failed: + print(f"[WARN] Could not delete {table}: {e}") + break + + remaining = [t for t, _ in failed] + + print("[OK] Cleanup complete") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/relationships.py b/examples/aio/advanced/relationships.py new file mode 100644 index 00000000..a4f9ecff --- /dev/null +++ b/examples/aio/advanced/relationships.py @@ -0,0 +1,401 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async Relationship Management Example for Dataverse SDK. + +Async equivalent of examples/advanced/relationships.py. + +This example demonstrates: +- Creating one-to-many relationships using the core SDK API +- Creating lookup fields using the convenience method +- Creating many-to-many relationships +- Querying and deleting relationships +- Working with relationship metadata types + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client +- pip install azure-identity +""" + +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, +) + + +# Simple logging helper +def log_call(description): + print(f"\n-> {description}") + + +async def delete_relationship_if_exists(client, schema_name): + """Delete a relationship by schema name if it exists.""" + rel = await client.tables.get_relationship(schema_name) + if rel: + rel_id = rel.relationship_id + if rel_id: + await client.tables.delete_relationship(rel_id) + print(f" (Cleaned up existing relationship: {schema_name})") + return True + return False + + +async def cleanup_previous_run(client): + """Clean up any resources from a previous run to make the example idempotent.""" + print("\n-> Checking for resources from previous runs...") + + # Known relationship names created by this example + relationships = [ + "new_Department_Employee", + "contact_new_employee_new_ManagerId", + "new_employee_project", + ] + + # Known table names created by this example + tables = ["new_Employee", "new_Department", "new_Project"] + + # Delete relationships first (required before tables can be deleted) + for rel_name in relationships: + try: + await delete_relationship_if_exists(client, rel_name) + except Exception as e: + print(f" [WARN] Could not delete relationship {rel_name}: {e}") + + # Delete tables + for table_name in tables: + try: + if await client.tables.get(table_name): + await client.tables.delete(table_name) + print(f" (Cleaned up existing table: {table_name})") + except Exception as e: + print(f" [WARN] Could not delete table {table_name}: {e}") + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry helper with exponential backoff.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" * Backoff succeeded after {retry_count} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); waited {total_delay}s total.") + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK - Async Relationship Management Example") + print("=" * 80) + + # ============================================================================ + # 1. SETUP & AUTHENTICATION + # ============================================================================ + print("\n" + "=" * 80) + print("1. Setup & Authentication") + print("=" * 80) + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_example(client) + finally: + await credential.close() + + +async def _run_example(client): + # Initialize relationship IDs to None for cleanup safety + rel_id_1 = None + rel_id_2 = None + rel_id_3 = None + + # ============================================================================ + # 2. CLEANUP PREVIOUS RUN (Idempotency) + # ============================================================================ + print("\n" + "=" * 80) + print("2. Cleanup Previous Run (Idempotency)") + print("=" * 80) + + await cleanup_previous_run(client) + + # ============================================================================ + # 3. CREATE SAMPLE TABLES + # ============================================================================ + print("\n" + "=" * 80) + print("3. Create Sample Tables") + print("=" * 80) + + # Create a parent table (Department) + log_call("Creating 'new_Department' table") + + dept_table = await backoff( + lambda: client.tables.create( + "new_Department", + { + "new_DepartmentCode": "string", + "new_Budget": "decimal", + }, + ) + ) + print(f"[OK] Created table: {dept_table['table_schema_name']}") + + # Create a child table (Employee) + log_call("Creating 'new_Employee' table") + + emp_table = await backoff( + lambda: client.tables.create( + "new_Employee", + { + "new_EmployeeNumber": "string", + "new_Salary": "decimal", + }, + ) + ) + print(f"[OK] Created table: {emp_table['table_schema_name']}") + + # Create a project table for many-to-many example + log_call("Creating 'new_Project' table") + + proj_table = await backoff( + lambda: client.tables.create( + "new_Project", + { + "new_ProjectCode": "string", + "new_StartDate": "datetime", + }, + ) + ) + print(f"[OK] Created table: {proj_table['table_schema_name']}") + + # ============================================================================ + # 4. CREATE ONE-TO-MANY RELATIONSHIP (Core SDK API) + # ============================================================================ + print("\n" + "=" * 80) + print("4. Create One-to-Many Relationship (Core API)") + print("=" * 80) + + log_call("Creating lookup field on Employee referencing Department") + + # Define the lookup attribute metadata + lookup = LookupAttributeMetadata( + schema_name="new_DepartmentId", + display_name=Label(localized_labels=[LocalizedLabel(label="Department", language_code=1033)]), + required_level="None", + ) + + # Define the relationship metadata + relationship = OneToManyRelationshipMetadata( + schema_name="new_Department_Employee", + referenced_entity=dept_table["table_logical_name"], + referencing_entity=emp_table["table_logical_name"], + referenced_attribute=f"{dept_table['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + merge=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + + # Create the relationship + result = await backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) + ) + + print(f"[OK] Created relationship: {result.relationship_schema_name}") + print(f" Lookup field: {result.lookup_schema_name}") + print(f" Relationship ID: {result.relationship_id}") + + rel_id_1 = result.relationship_id + + # ============================================================================ + # 5. CREATE LOOKUP FIELD (Convenience Method) + # ============================================================================ + print("\n" + "=" * 80) + print("5. Create Lookup Field (Convenience Method)") + print("=" * 80) + + log_call("Creating lookup field on Employee referencing Contact as Manager") + + # Use the convenience method for simpler scenarios + # An Employee has a Manager (who is a Contact in the system) + result2 = await backoff( + lambda: client.tables.create_lookup_field( + referencing_table=emp_table["table_logical_name"], + lookup_field_name="new_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The employee's direct manager", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + + print(f"[OK] Created lookup using convenience method: {result2.lookup_schema_name}") + print(f" Relationship: {result2.relationship_schema_name}") + + rel_id_2 = result2.relationship_id + + # ============================================================================ + # 6. CREATE MANY-TO-MANY RELATIONSHIP + # ============================================================================ + print("\n" + "=" * 80) + print("6. Create Many-to-Many Relationship") + print("=" * 80) + + log_call("Creating M:N relationship between Employee and Project") + + # Define many-to-many relationship + m2m_relationship = ManyToManyRelationshipMetadata( + schema_name="new_employee_project", + entity1_logical_name=emp_table["table_logical_name"], + entity2_logical_name=proj_table["table_logical_name"], + ) + + result3 = await backoff( + lambda: client.tables.create_many_to_many_relationship( + relationship=m2m_relationship, + ) + ) + + print(f"[OK] Created M:N relationship: {result3.relationship_schema_name}") + print(f" Relationship ID: {result3.relationship_id}") + + rel_id_3 = result3.relationship_id + + # ============================================================================ + # 7. QUERY RELATIONSHIP METADATA + # ============================================================================ + print("\n" + "=" * 80) + print("7. Query Relationship Metadata") + print("=" * 80) + + log_call("Retrieving 1:N relationship by schema name") + + rel_metadata = await client.tables.get_relationship("new_Department_Employee") + if rel_metadata: + print(f"[OK] Found relationship: {rel_metadata.relationship_schema_name}") + print(f" Type: {rel_metadata.relationship_type}") + print(f" Referenced Entity: {rel_metadata.referenced_entity}") + print(f" Referencing Entity: {rel_metadata.referencing_entity}") + else: + print(" Relationship not found") + + log_call("Retrieving M:N relationship by schema name") + + m2m_metadata = await client.tables.get_relationship("new_employee_project") + if m2m_metadata: + print(f"[OK] Found relationship: {m2m_metadata.relationship_schema_name}") + print(f" Type: {m2m_metadata.relationship_type}") + print(f" Entity 1: {m2m_metadata.entity1_logical_name}") + print(f" Entity 2: {m2m_metadata.entity2_logical_name}") + else: + print(" Relationship not found") + + # ============================================================================ + # 8. CLEANUP + # ============================================================================ + print("\n" + "=" * 80) + print("8. Cleanup") + print("=" * 80) + + cleanup = input("\nDelete created relationships and tables? (y/n): ").strip().lower() + + if cleanup == "y": + # Delete relationships first (required before deleting tables) + log_call("Deleting relationships") + try: + if rel_id_1: + await backoff(lambda: client.tables.delete_relationship(rel_id_1)) + print(f" [OK] Deleted relationship: new_Department_Employee") + except Exception as e: + print(f" [WARN] Error deleting relationship 1: {e}") + + try: + if rel_id_2: + await backoff(lambda: client.tables.delete_relationship(rel_id_2)) + print(f" [OK] Deleted relationship: contact->employee (Manager)") + except Exception as e: + print(f" [WARN] Error deleting relationship 2: {e}") + + try: + if rel_id_3: + await backoff(lambda: client.tables.delete_relationship(rel_id_3)) + print(f" [OK] Deleted relationship: new_employee_project") + except Exception as e: + print(f" [WARN] Error deleting relationship 3: {e}") + + # Delete tables + log_call("Deleting tables") + for table_name in ["new_Employee", "new_Department", "new_Project"]: + try: + await backoff(lambda name=table_name: client.tables.delete(name)) + print(f" [OK] Deleted table: {table_name}") + except Exception as e: + print(f" [WARN] Error deleting {table_name}: {e}") + + print("\n[OK] Cleanup complete") + else: + print("\nSkipping cleanup. Remember to manually delete:") + print(" - Relationships: new_Department_Employee, contact->employee (Manager), new_employee_project") + print(" - Tables: new_Employee, new_Department, new_Project") + + print("\n" + "=" * 80) + print("Example Complete!") + print("=" * 80) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\nExample interrupted by user.") + sys.exit(1) + except Exception as e: + print(f"\n\nError: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/examples/aio/advanced/sql_examples.py b/examples/aio/advanced/sql_examples.py new file mode 100644 index 00000000..4edd635c --- /dev/null +++ b/examples/aio/advanced/sql_examples.py @@ -0,0 +1,925 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async end-to-end SQL query examples -- pure SQL workflows in Dataverse. + +Async equivalent of examples/advanced/sql_examples.py. + +This example demonstrates everything a SQL developer can do through the +Python SDK's ``client.query.sql()`` and ``client.dataframe.sql()`` methods. + +See examples/advanced/sql_examples.py for the complete capability reference. + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import asyncio +import sys +import json +from collections import defaultdict +from enum import IntEnum + +import pandas as pd +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an async operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} " f"retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +class Region(IntEnum): + NORTH = 1 + SOUTH = 2 + EAST = 3 + WEST = 4 + + +async def main(): + print("=" * 80) + print("Dataverse SDK -- Async SQL End-to-End (Pure SQL Workflows)") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_examples(client) + finally: + await credential.close() + + +async def _run_examples(client): + parent_table = "new_SQLDemoTeam" + child_table = "new_SQLDemoTask" + + # ================================================================== + # 2. Seed demo data (SDK writes -- SQL is read-only) + # ================================================================== + heading(2, "Seed Demo Data (SDK Writes -- SQL Is Read-Only)") + print( + "[INFO] SQL is read-only (no INSERT/UPDATE/DELETE). We use the SDK's\n" + "records namespace to seed data, then query it all via SQL." + ) + + log_call(f"client.tables.get('{parent_table}')") + if await client.tables.get(parent_table): + print(f"[OK] Table already exists: {parent_table}") + else: + log_call(f"client.tables.create('{parent_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + parent_table, + { + "new_Code": "string", + "new_Region": Region, + "new_Budget": "decimal", + "new_Active": "bool", + }, + ) + ) + print(f"[OK] Created table: {parent_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {parent_table} (skipped)") + else: + raise + + log_call(f"client.tables.get('{child_table}')") + if await client.tables.get(child_table): + print(f"[OK] Table already exists: {child_table}") + else: + log_call(f"client.tables.create('{child_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + child_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {child_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {child_table} (skipped)") + else: + raise + + # Create lookup so tasks reference teams via JOIN + print("\n[INFO] Creating lookup field so tasks reference teams via JOIN...") + try: + await client.tables.create_lookup_field( + referencing_table=child_table, + lookup_field_name="new_TeamId", + referenced_table=parent_table, + display_name="Team", + ) + print("[OK] Created lookup: new_TeamId on tasks -> teams") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + log_call(f"client.records.create('{parent_table}', [...])") + teams = [ + {"new_Code": "ALPHA", "new_Region": Region.NORTH, "new_Budget": 50000, "new_Active": True}, + {"new_Code": "BRAVO", "new_Region": Region.SOUTH, "new_Budget": 75000, "new_Active": True}, + {"new_Code": "CHARLIE", "new_Region": Region.EAST, "new_Budget": 30000, "new_Active": False}, + {"new_Code": "DELTA", "new_Region": Region.WEST, "new_Budget": 90000, "new_Active": True}, + {"new_Code": "ECHO", "new_Region": Region.NORTH, "new_Budget": 42000, "new_Active": True}, + ] + team_ids = await backoff(lambda: client.records.create(parent_table, teams)) + print(f"[OK] Seeded {len(team_ids)} teams") + + parent_logical = parent_table.lower() + parent_set = f"{parent_logical}s" + try: + tinfo = await client.tables.get(parent_table) + if tinfo: + parent_set = tinfo.get("entity_set_name", parent_set) + except Exception: + pass + + log_call(f"client.records.create('{child_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[4]})", + }, + ] + task_ids = await backoff(lambda: client.records.create(child_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks (with team lookups)") + + parent_id_col = f"{parent_logical}id" + + try: + # ============================================================== + # 3. Schema discovery + # ============================================================== + heading(3, "Schema Discovery Before Writing SQL") + log_call(f"client.tables.list_columns('{parent_table}', select=[...])") + columns = await backoff( + lambda: client.tables.list_columns( + parent_table, + select=["LogicalName", "SchemaName", "AttributeType"], + ) + ) + custom_cols = [c for c in columns if c.get("LogicalName", "").startswith("new_")] + print(f"[OK] Custom columns on {parent_table}:") + for col in custom_cols: + print(f" {col['LogicalName']:30s} Type: {col.get('AttributeType', 'N/A')}") + + log_call(f"client.tables.list_table_relationships('{child_table}', ...)") + rels = await backoff( + lambda: client.tables.list_table_relationships( + child_table, + select=["SchemaName"], + ) + ) + print(f"[OK] Relationships on {child_table}: {len(rels)}") + + # ============================================================== + # 4. Basic SELECT + # ============================================================== + heading(4, "Basic SQL -- SELECT Specific Columns") + sql = f"SELECT new_code, new_budget, new_active FROM {parent_table}" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} rows:") + for r in results: + print(f" {r.get('new_code', ''):<12s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + + # ============================================================== + # 5. SELECT * -- Rejected by Design + # ============================================================== + heading(5, "SELECT * -- Rejected by Design") + print( + "SELECT * is deliberately rejected -- not a server workaround,\n" + "but an intentional design decision. Wide entities (e.g. account\n" + "has 307 columns) make SELECT * extremely expensive on shared\n" + "infrastructure. Specify columns explicitly instead.\n" + "Use client.query.sql_columns('account') to discover column names." + ) + from PowerPlatform.Dataverse.core.errors import ValidationError as _VE + + try: + await client.query.sql(f"SELECT * FROM {parent_table}") + print("[UNEXPECTED] SELECT * did not raise -- check SDK version") + except _VE as exc: + print(f"[OK] ValidationError raised as expected: {exc}") + + # ============================================================== + # 6. WHERE clause + # ============================================================== + heading(6, "SQL -- WHERE (=, >, <, IN, IS NULL, BETWEEN)") + sql = f"SELECT new_code, new_budget FROM {parent_table} WHERE new_budget > 40000" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] budget > 40000: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} WHERE new_code IN ('ALPHA', 'DELTA')" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] IN clause: {[r.get('new_code') for r in results]}") + + sql = f"SELECT new_title FROM {child_table} WHERE new_priority IS NOT NULL" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] IS NOT NULL: {len(results)} tasks") + + # ============================================================== + # 7. LIKE + # ============================================================== + heading(7, "SQL -- LIKE Pattern Matching") + sql = f"SELECT new_title FROM {child_table} WHERE new_title LIKE '%test%'" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] LIKE '%test%': {len(results)} matches") + + # ============================================================== + # 8. TOP + ORDER BY + # ============================================================== + heading(8, "SQL -- TOP N + ORDER BY") + sql = f"SELECT TOP 3 new_code, new_budget FROM {parent_table} ORDER BY new_budget DESC" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Top 3 by budget:") + for r in results: + print(f" {r.get('new_code', ''):<12s} Budget={r.get('new_budget')}") + + # ============================================================== + # 9. Aliases + # ============================================================== + heading(9, "SQL -- Table and Column Aliases") + sql = ( + f"SELECT t.new_code AS team_code, t.new_budget AS budget " + f"FROM {parent_table} AS t WHERE t.new_active = 1" + ) + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Aliased results: {len(results)} rows") + + # ============================================================== + # 10. DISTINCT + # ============================================================== + heading(10, "SQL -- DISTINCT") + sql = f"SELECT DISTINCT new_region FROM {parent_table}" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Distinct regions: {[r.get('new_region') for r in results]}") + + sql = f"SELECT DISTINCT TOP 2 new_region FROM {parent_table}" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] DISTINCT TOP 2: {[r.get('new_region') for r in results]}") + + # ============================================================== + # 11. Aggregates: COUNT, SUM, AVG, MIN, MAX + # ============================================================== + heading(11, "SQL -- Aggregates (All Run on Server)") + sql = ( + f"SELECT COUNT(*) as cnt, SUM(new_budget) as total, " + f"AVG(new_budget) as avg_b, MIN(new_budget) as min_b, " + f"MAX(new_budget) as max_b FROM {parent_table}" + ) + log_call('client.query.sql("SELECT COUNT, SUM, AVG, MIN, MAX...")') + results = await backoff(lambda: client.query.sql(sql)) + if results: + print(f"[OK] {json.dumps(dict(results[0]), indent=2)}") + + # ============================================================== + # 12. GROUP BY + # ============================================================== + heading(12, "SQL -- GROUP BY (Server-Side)") + sql = ( + f"SELECT new_region, COUNT(*) as team_count, " + f"SUM(new_budget) as total_budget " + f"FROM {parent_table} GROUP BY new_region" + ) + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} groups:") + for r in results: + print(f" Region={r.get('new_region')} Count={r.get('team_count')} Total={r.get('total_budget')}") + + # ============================================================== + # 13. INNER JOIN + # ============================================================== + heading(13, "SQL -- INNER JOIN") + print("Use the lookup attribute's logical name (e.g. new_teamid) for JOINs.") + + lookup_col = "new_teamid" # Lookup logical name, NOT _..._value + join_clause = f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_logical}id" + print(f"[INFO] Lookup column: {lookup_col}") + print(f"[INFO] JOIN clause: {join_clause}") + + sql = f"SELECT t.new_code, tk.new_title, tk.new_hours FROM {child_table} tk {join_clause}" + log_call('client.query.sql("...INNER JOIN...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] JOIN: {len(results)} rows") + for r in results[:5]: + print( + f" Team={r.get('new_code', ''):<10s} Task={r.get('new_title', ''):<25s} Hours={r.get('new_hours')}" + ) + except Exception as e: + print(f"[WARN] JOIN failed: {e}") + + # ============================================================== + # 14. LEFT JOIN + # ============================================================== + heading(14, "SQL -- LEFT JOIN") + sql = ( + f"SELECT t.new_code, tk.new_title " + f"FROM {parent_table} t " + f"LEFT JOIN {child_table} tk ON t.{parent_id_col} = tk.{lookup_col}" + ) + log_call('client.query.sql("...LEFT JOIN...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] LEFT JOIN: {len(results)} rows") + except Exception as e: + print(f"[WARN] LEFT JOIN failed: {e}") + + # ============================================================== + # 15. JOIN + GROUP BY + aggregates + # ============================================================== + heading(15, "SQL -- JOIN + GROUP BY + Aggregates") + sql = ( + f"SELECT t.new_code, COUNT(tk.new_sqldemotaskid) as task_count, " + f"SUM(tk.new_hours) as total_hours " + f"FROM {parent_table} t " + f"JOIN {child_table} tk ON t.{parent_id_col} = tk.{lookup_col} " + f"GROUP BY t.new_code" + ) + log_call('client.query.sql("...JOIN...GROUP BY...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} groups:") + for r in results: + print(f" Team={r.get('new_code', ''):<10s} Tasks={r.get('task_count')} Hours={r.get('total_hours')}") + except Exception as e: + print(f"[WARN] JOIN+GROUP BY failed: {e}") + + # ============================================================== + # 16. OFFSET FETCH (server-side pagination) + # ============================================================== + heading(16, "SQL -- OFFSET FETCH (Server-Side Pagination)") + page_size = 3 + for pg in range(1, 4): + offset = (pg - 1) * page_size + sql = ( + f"SELECT new_title, new_hours FROM {child_table} " + f"ORDER BY new_hours " + f"OFFSET {offset} ROWS FETCH NEXT {page_size} ROWS ONLY" + ) + log_call(f"Page {pg}: OFFSET {offset} FETCH NEXT {page_size}") + results = await backoff(lambda sql=sql: client.query.sql(sql)) + print(f" Page {pg}: {len(results)} rows") + for r in results: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + if len(results) < page_size: + break + + # ============================================================== + # 17. SQL to DataFrame + # ============================================================== + heading(17, "SQL to DataFrame (client.dataframe.sql)") + print("Get SQL results directly as a pandas DataFrame.") + sql = f"SELECT new_code, new_budget, new_region " f"FROM {parent_table} ORDER BY new_budget DESC" + log_call(f'client.dataframe.sql("{sql}")') + df = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] DataFrame: {len(df)} rows x {len(df.columns)} columns") + print(df.to_string(index=False)) + print(f"\n Mean budget: {df['new_budget'].mean():,.2f}") + print(f" Budget by region:\n{df.groupby('new_region')['new_budget'].sum()}") + + # ============================================================== + # 18. SQL to DataFrame with JOINs + # ============================================================== + heading(18, "SQL to DataFrame -- JOIN Query") + sql = ( + f"SELECT t.new_code, tk.new_title, tk.new_hours " + f"FROM {child_table} tk " + f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_id_col}" + ) + log_call('client.dataframe.sql("...JOIN...")') + try: + df_j = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] {len(df_j)} rows") + print(df_j.to_string(index=False)) + print("\n-- Pivot: hours by team --") + print(df_j.groupby("new_code")["new_hours"].agg(["sum", "mean", "count"]).to_string()) + except Exception as e: + print(f"[WARN] {e}") + + # ============================================================== + # 19. Built-in table JOINs + # ============================================================== + heading(19, "Built-In Table JOINs (account -> contact)") + sql = "SELECT a.name, c.fullname FROM account a " "INNER JOIN contact c ON a.accountid = c.parentcustomerid" + log_call('client.query.sql("...account JOIN contact...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} rows") + for r in results[:5]: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('fullname', '')}") + except Exception as e: + print(f"[INFO] {e}") + + # ============================================================== + # 20. LIMITATION: Writes require SDK + # ============================================================== + heading(20, "LIMITATION: Writes Require SDK (Read-Only SQL)") + sql = f"SELECT new_sqldemotaskid, new_title " f"FROM {child_table} WHERE new_done = 0" + incomplete = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] SQL found {len(incomplete)} incomplete tasks") + if incomplete: + fid = incomplete[0].get("new_sqldemotaskid") + if fid: + await backoff(lambda: client.records.update(child_table, fid, {"new_Done": True})) + print(f"[OK] Updated via SDK: '{incomplete[0].get('new_title')}'") + + # ============================================================== + # 21. LIMITATION: No subqueries + # ============================================================== + heading(21, "LIMITATION: No Subqueries -- Chain SQL Calls") + sql1 = f"SELECT {parent_id_col} FROM {parent_table} WHERE new_budget > 50000" + big = await backoff(lambda: client.query.sql(sql1)) + big_ids = [r.get(parent_id_col) for r in big if r.get(parent_id_col)] + print(f"[OK] Step 1: {len(big_ids)} teams with budget > 50000") + if big_ids: + id_list = ", ".join(f"'{i}'" for i in big_ids) + sql2 = f"SELECT new_title FROM {child_table} " f"WHERE {lookup_col} IN ({id_list})" + tasks_r = await backoff(lambda: client.query.sql(sql2)) + print(f"[OK] Step 2: {len(tasks_r)} tasks for big-budget teams") + + # ============================================================== + # 22. LIMITATION: No functions + # ============================================================== + heading(22, "LIMITATION: No Functions -- Post-Process in Python") + sql = f"SELECT new_code, new_budget FROM {parent_table}" + rows = await backoff(lambda: client.query.sql(sql)) + print("[OK] Post-processing (CASE equivalent):") + for r in rows: + b = float(r.get("new_budget") or 0) + tier = "HIGH" if b > 60000 else "MEDIUM" if b > 35000 else "LOW" + print(f" {r.get('new_code', ''):<12s} Budget={b:>10,.2f} Tier={tier}") + + # ============================================================== + # 23. Polymorphic lookups via SQL (ownerid, customerid) + # ============================================================== + heading(23, "Polymorphic Lookups via SQL (ownerid, customerid)") + print( + "Some Dataverse lookup columns are POLYMORPHIC -- the GUID can\n" + "point to different entity types (e.g. ownerid -> systemuser OR\n" + "team, customerid -> account OR contact).\n" + "\n" + "SQL pattern: INNER JOIN acts as both a join AND a type filter.\n" + "If the GUID points to a different type, the JOIN simply returns\n" + "no row -- so you get exactly the records of the type you joined." + ) + + # 23a. Discover lookup columns on a table + print("\n-- 23a. Discover lookup columns on account --") + log_call("client.tables.list_columns('account', filter=Lookup)") + try: + acct_cols = await backoff( + lambda: client.tables.list_columns( + "account", + select=["LogicalName", "AttributeType"], + filter="AttributeType eq 'Lookup' or AttributeType eq 'Owner' or AttributeType eq 'Customer'", + ) + ) + lookup_names = sorted(c.get("LogicalName", "") for c in acct_cols if c.get("LogicalName", "")) + print(f"[OK] Lookup columns on account ({len(lookup_names)} found):") + for ln in lookup_names[:10]: + print(f" {ln}") + if len(lookup_names) > 10: + print(f" ... and {len(lookup_names) - 10} more") + except Exception as e: + print(f"[INFO] Lookup discovery skipped: {e}") + + # 23b. Discover polymorphic targets via relationship metadata + print("\n-- 23b. Discover which entities a polymorphic lookup targets --") + log_call("client.tables.list_table_relationships('account', ...)") + try: + acct_rels = await backoff(lambda: client.tables.list_table_relationships("account")) + by_attr = defaultdict(list) + for rel in acct_rels: + attr = rel.get("ReferencingAttribute", "") + ref = rel.get("ReferencedEntity", "") + if attr and ref and rel.get("ReferencingEntity", "").lower() == "account": + by_attr[attr].append(ref) + print("[OK] Lookup targets on account:") + for attr, targets in sorted(by_attr.items()): + tag = "POLYMORPHIC" if len(targets) > 1 else "regular" + print(f" {attr:<35s} -> {', '.join(targets):<30s} [{tag}]") + except Exception as e: + print(f"[INFO] Relationship discovery skipped: {e}") + + # 23c. Resolve ownerid (polymorphic: systemuser or team) + print("\n-- 23c. Resolve ownerid via SQL JOINs --") + print("ownerid is polymorphic (systemuser or team). Use separate\n" "JOINs and combine in a DataFrame.") + try: + # Records owned by users + log_call("SQL: account JOIN systemuser ON ownerid") + df_user_owned = await backoff( + lambda: client.dataframe.sql( + "SELECT TOP 5 a.name, su.fullname as owner_name " + "FROM account a " + "INNER JOIN systemuser su ON a.ownerid = su.systemuserid" + ) + ) + df_user_owned["owner_type"] = "User" + + # Records owned by teams + log_call("SQL: account JOIN team ON ownerid") + df_team_owned = await backoff( + lambda: client.dataframe.sql( + "SELECT TOP 5 a.name, t.name as owner_name " + "FROM account a " + "INNER JOIN team t ON a.ownerid = t.teamid" + ) + ) + df_team_owned["owner_type"] = "Team" + + df_owners = pd.concat([df_user_owned, df_team_owned], ignore_index=True) + print(f"[OK] Owner resolution: {len(df_owners)} rows") + print(f" User-owned: {len(df_user_owned)}") + print(f" Team-owned: {len(df_team_owned)}") + if not df_owners.empty: + print(df_owners.to_string(index=False)) + except Exception as e: + print(f"[INFO] Owner resolution skipped (may have no data): {e}") + + # 23d. Track created-by and modified-by (common audit pattern) + print("\n-- 23d. Audit trail: who created/modified records (via SQL) --") + try: + log_call("SQL: account JOIN systemuser (createdby + modifiedby)") + results = await backoff( + lambda: client.query.sql( + "SELECT TOP 5 a.name, " + "creator.fullname as created_by, " + "modifier.fullname as modified_by " + "FROM account a " + "JOIN systemuser creator ON a.createdby = creator.systemuserid " + "JOIN systemuser modifier ON a.modifiedby = modifier.systemuserid" + ) + ) + print(f"[OK] Audit trail: {len(results)} rows") + for r in results[:5]: + print( + f" {r.get('name', ''):<25s} " + f"Created: {r.get('created_by', ''):<20s} " + f"Modified: {r.get('modified_by', '')}" + ) + except Exception as e: + print(f"[INFO] Audit trail skipped: {e}") + + # ============================================================== + # 24. SQL Read -> DataFrame Transform -> SDK Write-Back + # ============================================================== + heading(24, "SQL Read -> DataFrame Transform -> SDK Write-Back") + print( + "The full bidirectional workflow for SQL users:\n" + " 1. SQL query -> DataFrame (read)\n" + " 2. pandas -> Transform (compute)\n" + " 3. DataFrame -> SDK write-back (create/update/delete)\n" + "\n" + "This is how SQL developers do end-to-end work without\n" + "learning OData or the Web API." + ) + + # Read current state via SQL + sql = f"SELECT new_sqldemotaskid, new_title, new_hours, new_done " f"FROM {child_table}" + log_call(f'client.dataframe.sql("{sql}")') + df_tasks = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] Read {len(df_tasks)} tasks via SQL") + print(df_tasks.to_string(index=False)) + + # Transform: bump hours by 1 for incomplete tasks + mask = df_tasks["new_done"] == False # noqa: E712 + original_hours = df_tasks.loc[mask, "new_hours"].copy() + df_tasks.loc[mask, "new_hours"] = df_tasks.loc[mask, "new_hours"] + 1 + changed = mask.sum() + print(f"\n[OK] Bumped hours +1 for {changed} incomplete tasks (in DataFrame)") + + # Write back via SDK + if changed > 0: + updates = df_tasks.loc[mask, ["new_sqldemotaskid", "new_hours"]] + log_call(f"client.dataframe.update('{child_table}', ..., id_column='new_sqldemotaskid')") + await backoff(lambda: client.dataframe.update(child_table, updates, id_column="new_sqldemotaskid")) + print(f"[OK] Wrote back {len(updates)} updated rows via DataFrame") + + # Verify with SQL + verify = await backoff( + lambda: client.dataframe.sql(f"SELECT new_title, new_hours FROM {child_table} WHERE new_done = 0") + ) + print(f"[OK] Verified via SQL -- incomplete tasks now:") + print(verify.to_string(index=False)) + + # Restore original values + df_tasks.loc[mask, "new_hours"] = original_hours + restore = df_tasks.loc[mask, ["new_sqldemotaskid", "new_hours"]] + await backoff(lambda: client.dataframe.update(child_table, restore, id_column="new_sqldemotaskid")) + print("[OK] Restored original hours") + + # ============================================================== + # 25. SQL-driven bulk create from query results + # ============================================================== + heading(25, "SQL-Driven Bulk Create (Query -> Transform -> Insert)") + print( + "Pattern: query existing data with SQL, transform it,\n" + "then create new records via DataFrame -- all without\n" + "learning OData syntax." + ) + + # Read teams via SQL + sql = f"SELECT new_code, new_budget FROM {parent_table} WHERE new_active = 1" + log_call(f'client.dataframe.sql("{sql}")') + df_active = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] Read {len(df_active)} active teams via SQL") + + # Transform: create a new task for each active team + new_tasks = pd.DataFrame( + { + "new_Title": [f"Review budget for {code}" for code in df_active["new_code"]], + "new_Hours": [2] * len(df_active), + "new_Done": [False] * len(df_active), + "new_Priority": [1] * len(df_active), + } + ) + log_call(f"client.dataframe.create('{child_table}', DataFrame({len(new_tasks)} rows))") + new_ids = await backoff(lambda: client.dataframe.create(child_table, new_tasks)) + print(f"[OK] Created {len(new_ids)} new tasks from SQL query results") + + # Verify with SQL + verify_sql = f"SELECT new_title, new_hours FROM {child_table} " f"WHERE new_title LIKE 'Review budget%'" + created_tasks = await backoff(lambda: client.query.sql(verify_sql)) + print(f"[OK] Verified via SQL: {len(created_tasks)} 'Review budget' tasks") + + # Clean up the created tasks + await backoff(lambda: client.dataframe.delete(child_table, new_ids)) + print(f"[OK] Cleaned up {len(new_ids)} demo tasks") + + # ============================================================== + # 26. SQL-driven bulk delete + # ============================================================== + heading(26, "SQL-Driven Bulk Delete (Query -> Filter -> Delete)") + print("Pattern: find records with SQL, filter in pandas,\n" "then delete via DataFrame -- pure SQL thinking.") + + # Create some temp records to demonstrate + temp = pd.DataFrame( + { + "new_Title": ["TEMP: delete me 1", "TEMP: delete me 2", "TEMP: keep me"], + "new_Hours": [1, 2, 3], + "new_Done": [False, False, False], + "new_Priority": [1, 1, 1], + } + ) + temp_ids = await backoff(lambda: client.dataframe.create(child_table, temp)) + print(f"[OK] Created {len(temp_ids)} temp records") + + # SQL to find, pandas to filter, SDK to delete + sql = f"SELECT new_sqldemotaskid, new_title FROM {child_table} WHERE new_title LIKE 'TEMP:%'" + df_temp = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] SQL found {len(df_temp)} TEMP records") + + # Filter in pandas: only delete the "delete me" ones + to_delete = df_temp[df_temp["new_title"].str.contains("delete me")] + print(f"[OK] Pandas filtered to {len(to_delete)} records to delete") + + if not to_delete.empty: + log_call("client.dataframe.delete(...)") + await backoff(lambda: client.dataframe.delete(child_table, to_delete["new_sqldemotaskid"])) + print(f"[OK] Deleted {len(to_delete)} records via DataFrame") + + # Verify the "keep me" record survived + remaining = await backoff( + lambda: client.query.sql(f"SELECT new_title FROM {child_table} WHERE new_title LIKE 'TEMP:%'") + ) + print(f"[OK] Remaining TEMP records: {len(remaining)}") + for r in remaining: + print(f" {r.get('new_title')}") + + # Clean up the surviving temp record + keep_ids = [ + r.get("new_sqldemotaskid") + for r in await backoff( + lambda: client.query.sql(f"SELECT new_sqldemotaskid FROM {child_table} WHERE new_title LIKE 'TEMP:%'") + ) + if r.get("new_sqldemotaskid") + ] + for kid in keep_ids: + await backoff(lambda kid=kid: client.records.delete(child_table, kid)) + + # ============================================================== + # 27. AND/OR, NOT IN, NOT LIKE + # ============================================================== + heading(27, "SQL -- AND/OR, NOT IN, NOT LIKE") + sql = f"SELECT new_code, new_budget FROM {parent_table} " f"WHERE new_active = 1 AND new_budget > 40000" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] AND: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} " f"WHERE new_code = 'ALPHA' OR new_code = 'DELTA'" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] OR: {[r.get('new_code') for r in results]}") + + sql = ( + f"SELECT new_code FROM {parent_table} " + f"WHERE new_active = 1 AND (new_budget > 80000 OR new_budget < 45000)" + ) + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] AND + OR with parens: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} WHERE new_code NOT IN ('ALPHA')" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] NOT IN: {[r.get('new_code') for r in results]}") + + sql = f"SELECT new_title FROM {child_table} WHERE new_title NOT LIKE 'Design%'" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] NOT LIKE: {len(results)} rows") + + # ============================================================== + # 28. Deep JOINs (5-8 tables) + # ============================================================== + heading(28, "Deep JOINs (5+ Tables) -- No Depth Limit") + + sql = ( + "SELECT TOP 3 a.name, c.fullname, o.name as opp, " + "su.fullname as owner, bu.name as bu " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "JOIN opportunity o ON a.accountid = o.parentaccountid " + "JOIN systemuser su ON a.ownerid = su.systemuserid " + "JOIN businessunit bu ON su.businessunitid = bu.businessunitid" + ) + log_call("5-table JOIN") + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] 5-table JOIN: {len(results)} rows") + except Exception as e: + print(f"[INFO] {e}") + + # ============================================================== + # 29. SQL Helper Functions + # ============================================================== + heading(29, "SQL Helper Functions (query.sql_*)") + print( + "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" + "sql_select(), sql_join(), and sql_joins() were removed -- write JOIN\n" + "clauses directly or use client.query.fetchxml() for complex queries." + ) + + # sql_columns — still available at GA + log_call(f"client.query.sql_columns('{parent_table}')") + cols = await client.query.sql_columns(parent_table) + print(f"[OK] {len(cols)} columns:") + for c in cols[:5]: + print(f" {c['name']:30s} Type: {c['type']:15s} PK={c['is_pk']}") + + # ============================================================== + # 30. OData Helper Functions + # ============================================================== + heading(30, "OData Helper Functions (query.odata_expands)") + print( + "odata_expands() is available without deprecation.\n" + "odata_select(), odata_expand(), and odata_bind() are deprecated\n" + "at GA -- use the typed query builder instead." + ) + + # odata_expands + log_call(f"client.query.odata_expands('{child_table}')") + try: + expands = await client.query.odata_expands(child_table) + print(f"[OK] {len(expands)} expand targets:") + for e in expands[:5]: + print(f" nav={e['nav_property']:30s} -> {e['target_table']}") + except Exception as e: + print(f"[WARN] {e}") + + finally: + heading(31, "Cleanup") + for tbl in [child_table, parent_table]: + log_call(f"client.tables.delete('{tbl}')") + try: + await backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + code = getattr(getattr(ex, "response", None), "status_code", None) + if isinstance(ex, MetadataError) and code == 404: + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("Async SQL Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/walkthrough.py b/examples/aio/advanced/walkthrough.py new file mode 100644 index 00000000..d7a14e4a --- /dev/null +++ b/examples/aio/advanced/walkthrough.py @@ -0,0 +1,632 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async walkthrough demonstrating core Dataverse SDK operations. + +Async equivalent of examples/advanced/walkthrough.py. + +This example shows: +- Table creation with various column types including enums +- Single and multiple record CRUD operations +- Querying with filtering, paging, AsyncQueryBuilder, and SQL +- Expand (navigation properties) with AsyncQueryBuilder +- Picklist label-to-value conversion +- Column management +- Batch operations (create, read, update, changeset, delete in one HTTP request) +- Cleanup + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client +- pip install azure-identity +""" + +import asyncio +import sys +import json +from enum import IntEnum + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + +def log_call(description): + print(f"\n-> {description}") + + +class Priority(IntEnum): + LOW = 1 + MEDIUM = 2 + HIGH = 3 + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off for metadata propagation delays.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print(f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total.") + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK Async Walkthrough") + print("=" * 80) + + # ============================================================================ + # 1. SETUP & AUTHENTICATION + # ============================================================================ + print("\n" + "=" * 80) + print("1. Setup & Authentication") + print("=" * 80) + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_walkthrough(client) + finally: + await credential.close() + + +async def _run_walkthrough(client): + # ============================================================================ + # 2. TABLE CREATION (METADATA) + # ============================================================================ + print("\n" + "=" * 80) + print("2. Table Creation (Metadata)") + print("=" * 80) + + table_name = "new_WalkthroughDemo" + + log_call(f"await client.tables.get('{table_name}')") + table_info = await backoff(lambda: client.tables.get(table_name)) + + if table_info: + print(f"[OK] Table already exists: {table_info.get('table_schema_name')}") + print(f" Logical Name: {table_info.get('table_logical_name')}") + print(f" Entity Set: {table_info.get('entity_set_name')}") + else: + log_call(f"await client.tables.create('{table_name}', columns={{...}}, display_name='Walkthrough Demo')") + columns = { + "new_Title": "string", + "new_Quantity": "int", + "new_Amount": "decimal", + "new_Completed": "bool", + "new_Notes": "memo", + "new_Priority": Priority, + } + table_info = await backoff(lambda: client.tables.create(table_name, columns, display_name="Walkthrough Demo")) + print(f"[OK] Created table: {table_info.get('table_schema_name')}") + print(f" Columns created: {', '.join(table_info.get('columns_created', []))}") + + # ============================================================================ + # 3. CREATE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("3. Create Operations") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', {{...}})") + single_record = { + "new_Title": "Complete project documentation", + "new_Quantity": 5, + "new_Amount": 1250.50, + "new_Completed": False, + "new_Notes": "This is a multiline memo field.\nIt supports longer text content.", + "new_Priority": Priority.MEDIUM, + } + id1 = await backoff(lambda: client.records.create(table_name, single_record)) + print(f"[OK] Created single record: {id1}") + + log_call(f"await client.records.create('{table_name}', [{{...}}, {{...}}, {{...}}])") + multiple_records = [ + { + "new_Title": "Review code changes", + "new_Quantity": 10, + "new_Amount": 500.00, + "new_Completed": True, + "new_Priority": Priority.HIGH, + }, + { + "new_Title": "Update test cases", + "new_Quantity": 8, + "new_Amount": 750.25, + "new_Completed": False, + "new_Priority": Priority.LOW, + }, + { + "new_Title": "Deploy to staging", + "new_Quantity": 3, + "new_Amount": 2000.00, + "new_Completed": False, + "new_Priority": Priority.HIGH, + }, + ] + ids = await backoff(lambda: client.records.create(table_name, multiple_records)) + print(f"[OK] Created {len(ids)} records: {ids}") + + # ============================================================================ + # 4. READ OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("4. Read Operations") + print("=" * 80) + + log_call(f"await client.records.retrieve('{table_name}', '{id1}')") + record = await backoff(lambda: client.records.retrieve(table_name, id1)) + print("[OK] Retrieved single record:") + print( + json.dumps( + { + "new_walkthroughdemoid": record.get("new_walkthroughdemoid"), + "new_title": record.get("new_title"), + "new_quantity": record.get("new_quantity"), + "new_amount": record.get("new_amount"), + "new_completed": record.get("new_completed"), + "new_notes": record.get("new_notes"), + "new_priority": record.get("new_priority"), + "new_priority@FormattedValue": record.get("new_priority@OData.Community.Display.V1.FormattedValue"), + }, + indent=2, + ) + ) + + log_call(f"await client.records.list('{table_name}', filter='new_quantity gt 5')") + all_records = await backoff(lambda: client.records.list(table_name, filter="new_quantity gt 5")) + print(f"[OK] Found {len(all_records)} records with new_quantity > 5") + for rec in all_records: + print(f" - new_Title='{rec.get('new_title')}', new_Quantity={rec.get('new_quantity')}") + + # ============================================================================ + # 5. UPDATE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("5. Update Operations") + print("=" * 80) + + log_call(f"await client.records.update('{table_name}', '{id1}', {{...}})") + await backoff( + lambda: client.records.update( + table_name, + id1, + { + "new_Quantity": 100, + "new_Notes": "Updated memo field.\nNow with revised content across multiple lines.", + }, + ) + ) + updated = await backoff(lambda: client.records.retrieve(table_name, id1)) + print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") + print(f" new_Notes: {repr(updated.get('new_notes'))}") + + log_call(f"await client.records.update('{table_name}', [{len(ids)} IDs], {{...}})") + await backoff(lambda: client.records.update(table_name, ids, {"new_Completed": True})) + print(f"[OK] Updated {len(ids)} records to new_Completed=True") + + # ============================================================================ + # 6. PAGING DEMO + # ============================================================================ + print("\n" + "=" * 80) + print("6. Paging Demo") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', [20 records])") + paging_records = [ + { + "new_Title": f"Paging test item {i}", + "new_Quantity": i, + "new_Amount": i * 10.0, + "new_Completed": False, + "new_Priority": Priority.LOW, + } + for i in range(1, 21) + ] + paging_ids = await backoff(lambda: client.records.create(table_name, paging_records)) + print(f"[OK] Created {len(paging_ids)} records for paging demo") + + log_call(f"async for page in client.query.builder('{table_name}').order_by().page_size(5).execute_pages()") + print("Fetching records with page_size=5...") + page_num = 0 + async for page in client.query.builder(table_name).order_by("new_Quantity").page_size(5).execute_pages(): + page_num += 1 + record_ids = [r.get("new_walkthroughdemoid")[:8] + "..." for r in page] + print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") + + # ============================================================================ + # 7. QUERYBUILDER - FLUENT QUERIES + # ============================================================================ + print("\n" + "=" * 80) + print("7. AsyncQueryBuilder - Fluent Queries") + print("=" * 80) + + log_call("await client.query.builder(...).select().where(col(...)==...).order_by().execute()") + print("Querying incomplete records ordered by amount (fluent builder)...") + qb_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Priority") + .where(col("new_Completed") == False) + .order_by("new_Amount", descending=True) + .top(10) + .execute() + ) + print(f"[OK] AsyncQueryBuilder found {len(qb_result)} incomplete records:") + for rec in list(qb_result)[:5]: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") + + log_call("await client.query.builder(...).where(col('new_Priority').in_([HIGH, LOW])).execute()") + print("Querying records with HIGH or LOW priority (col().in_())...") + priority_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Priority") + .where(col("new_Priority").in_([Priority.HIGH, Priority.LOW])) + .execute() + ) + print(f"[OK] Found {len(priority_result)} records with HIGH or LOW priority") + + log_call("await client.query.builder(...).where(col('new_Amount').between(500, 1500)).execute()") + range_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount") + .where(col("new_Amount").between(500, 1500)) + .execute() + ) + print(f"[OK] Found {len(range_result)} records with amount in [500, 1500]") + + log_call("await client.query.builder(...).where((col(...)==...) & (col(...) > ...)).execute()") + expr_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Quantity") + .where((col("new_Completed") == False) & (col("new_Amount") > 100)) + .order_by("new_Amount", descending=True) + .top(5) + .execute() + ) + print(f"[OK] Expression tree query found {len(expr_result)} records:") + for rec in expr_result: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") + + log_call("async for page in client.query.builder(...).where(...).page_size().execute_pages()") + print("Querying with combined expression filters and paging...") + combined_page_count = 0 + combined_record_count = 0 + async for page in ( + client.query.builder(table_name) + .select("new_Title", "new_Quantity") + .where(col("new_Completed") == False) + .where(col("new_Quantity").between(1, 15)) + .order_by("new_Quantity") + .page_size(3) + .execute_pages() + ): + combined_page_count += 1 + combined_record_count += len(page) + titles = [r.get("new_title", "?") for r in page] + print(f" Page {combined_page_count}: {len(page)} records - {titles}") + print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") + + log_call(f"(await client.query.builder('{table_name}').select(...).where(...).execute()).to_dataframe()") + print("Querying completed records as a pandas DataFrame (to_dataframe)...") + completed_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_title", "new_quantity") + .where(col("new_completed") == True) + .execute() + ) + df = completed_result.to_dataframe() + print(f"[OK] to_dataframe() returned {len(df)} rows, columns: {list(df.columns)}") + if not df.empty: + print(f" First row: new_title='{df.iloc[0].get('new_title')}', new_quantity={df.iloc[0].get('new_quantity')}") + print(f" Sum of new_quantity: {df['new_quantity'].sum()}") + else: + print(" (empty DataFrame)") + + # ============================================================================ + # 8. EXPAND (NAVIGATION PROPERTIES) + # ============================================================================ + print("\n" + "=" * 80) + print("8. Expand (Navigation Properties)") + print("=" * 80) + + log_call("await client.query.builder('account').select('name').expand('primarycontactid').top(3).execute()") + try: + expanded_records = await backoff( + lambda: client.query.builder("account").select("name").expand("primarycontactid").top(3).execute() + ) + print(f"[OK] Found {len(expanded_records)} accounts with expanded contact:") + for rec in expanded_records: + contact = rec.get("primarycontactid") + contact_name = contact.get("fullname", "(none)") if contact else "(no contact)" + print(f" - '{rec.get('name')}' -> Contact: {contact_name}") + except Exception as e: + print(f"[SKIP] Expand demo skipped (no accounts in org): {e}") + + log_call("ExpandOption('Account_Tasks').select('subject').order_by('createdon', descending=True).top(3)") + try: + tasks_opt = ( + ExpandOption("Account_Tasks").select("subject", "createdon").order_by("createdon", descending=True).top(3) + ) + nested_records = await backoff( + lambda: client.query.builder("account").select("name").expand(tasks_opt).top(3).execute() + ) + print(f"[OK] Found {len(nested_records)} accounts with nested task expansion:") + for rec in nested_records: + tasks = rec.get("Account_Tasks", []) + print(f" - '{rec.get('name')}' has {len(tasks)} task(s)") + except Exception as e: + print(f"[SKIP] Nested expand demo skipped: {e}") + + # ============================================================================ + # 9. SQL QUERY + # ============================================================================ + print("\n" + "=" * 80) + print("9. SQL Query") + print("=" * 80) + + sql = "SELECT new_title, new_quantity FROM new_walkthroughdemo WHERE new_completed = 1" + log_call(f"await client.query.sql('{sql}')") + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] SQL query returned {len(results)} completed records:") + for result in results[:5]: + print(f" - new_Title='{result.get('new_title')}', new_Quantity={result.get('new_quantity')}") + except Exception as e: + print(f"[WARN] SQL query failed: {str(e)}") + + # ============================================================================ + # 10. FETCHXML QUERY + # ============================================================================ + print("\n" + "=" * 80) + print("10. FetchXML Query") + print("=" * 80) + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(xml).execute()") + try: + fx_result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] FetchXML returned {len(fx_result)} incomplete records:") + for r in fx_result[:5]: + print(f" - '{r.get('new_title')}' Quantity={r.get('new_quantity')}") + except Exception as e: + print(f"[WARN] FetchXML query failed: {e}") + + log_call("async for page in client.query.fetchxml(paged_xml).execute_pages()") + paged_xml = f""" + + + + + + + """ + try: + fx_page_num = 0 + fx_total = 0 + async for page in client.query.fetchxml(paged_xml).execute_pages(): + fx_page_num += 1 + fx_total += len(page) + titles = [r.get("new_title", "?") for r in page] + print(f" Page {fx_page_num}: {len(page)} record(s) — {titles}") + print(f"[OK] FetchXML execute_pages(): {fx_total} total records across {fx_page_num} page(s)") + except Exception as e: + print(f"[WARN] FetchXML execute_pages failed: {e}") + + # ============================================================================ + # 11. PICKLIST LABEL CONVERSION + # ============================================================================ + print("\n" + "=" * 80) + print("11. Picklist Label Conversion") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', {{'new_Priority': 'High'}})") + label_record = { + "new_Title": "Test label conversion", + "new_Quantity": 1, + "new_Amount": 99.99, + "new_Completed": False, + "new_Priority": "High", + } + label_id = await backoff(lambda: client.records.create(table_name, label_record)) + retrieved = await backoff(lambda: client.records.retrieve(table_name, label_id)) + print(f"[OK] Created record with string label 'High' for new_Priority") + print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") + print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") + + log_call(f"await client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") + await backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) + updated_label = await backoff(lambda: client.records.retrieve(table_name, label_id)) + print(f"[OK] Updated record with string label 'Low' for new_Priority") + print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") + + # ============================================================================ + # 12. COLUMN MANAGEMENT + # ============================================================================ + print("\n" + "=" * 80) + print("12. Column Management") + print("=" * 80) + + log_call(f"await client.tables.add_columns('{table_name}', {{'new_Tags': 'string'}})") + created_cols = await backoff(lambda: client.tables.add_columns(table_name, {"new_Tags": "string"})) + print(f"[OK] Added column: {created_cols[0]}") + + log_call(f"await client.tables.remove_columns('{table_name}', ['new_Tags'])") + await backoff(lambda: client.tables.remove_columns(table_name, ["new_Tags"])) + print("[OK] Deleted column: new_Tags") + + # ============================================================================ + # 13. DELETE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("13. Delete Operations") + print("=" * 80) + + log_call(f"await client.records.delete('{table_name}', '{id1}')") + await backoff(lambda: client.records.delete(table_name, id1)) + print(f"[OK] Deleted single record: {id1}") + + log_call(f"await client.records.delete('{table_name}', [{len(paging_ids)} IDs])") + job_id = await backoff(lambda: client.records.delete(table_name, paging_ids)) + print(f"[OK] Bulk delete job started: {job_id}") + + # ============================================================================ + # 14. BATCH OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("14. Batch Operations") + print("=" * 80) + + log_call("client.batch.new() + batch.records.create(...) x2 + await batch.execute()") + batch = client.batch.new() + batch.records.create( + table_name, + { + "new_Title": "Batch task alpha", + "new_Quantity": 1, + "new_Amount": 25.0, + "new_Completed": False, + "new_Priority": Priority.LOW, + }, + ) + batch.records.create( + table_name, + { + "new_Title": "Batch task beta", + "new_Quantity": 2, + "new_Amount": 50.0, + "new_Completed": False, + "new_Priority": Priority.MEDIUM, + }, + ) + result = await batch.execute() + batch_ids = list(result.entity_ids) + print(f"[OK] Batch create: {len(result.succeeded)} operations, {len(batch_ids)} records created") + + log_call("client.batch.new() + batch.records.retrieve(...) x2 + await batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.retrieve(table_name, bid, select=["new_title", "new_quantity"]) + result = await batch.execute() + print(f"[OK] Batch get: {len(result.succeeded)} reads in one HTTP request") + for resp in result.succeeded: + if resp.data: + print(f" new_title='{resp.data.get('new_title')}', new_quantity={resp.data.get('new_quantity')}") + + log_call("async with batch.changeset() as cs: cs.records.create(...); cs.records.update(ref, ...)") + batch = client.batch.new() + async with batch.changeset() as cs: + cs_ref = cs.records.create( + table_name, + { + "new_Title": "Changeset task", + "new_Quantity": 5, + "new_Amount": 100.0, + "new_Completed": False, + "new_Priority": Priority.HIGH, + }, + ) + cs.records.update(table_name, cs_ref, {"new_Completed": True}) + result = await batch.execute() + if not result.has_errors: + batch_ids.extend(result.entity_ids) + print(f"[OK] Changeset: {len(result.succeeded)} operations committed atomically") + else: + for item in result.failed: + print(f"[WARN] Changeset error {item.status_code}: {item.error_message}") + + log_call(f"client.batch.new() + batch.records.delete(...) x{len(batch_ids)} + await batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.delete(table_name, bid) + result = await batch.execute(continue_on_error=True) + print(f"[OK] Batch delete: {len(result.succeeded)} records deleted in one HTTP request") + + # ============================================================================ + # 15. CLEANUP + # ============================================================================ + print("\n" + "=" * 80) + print("15. Cleanup") + print("=" * 80) + + log_call(f"await client.tables.delete('{table_name}')") + try: + await backoff(lambda: client.tables.delete(table_name)) + print(f"[OK] Deleted table: {table_name}") + except MetadataError as ex: + if "not found" in str(ex).lower(): + print(f"[OK] Table already removed: {table_name}") + else: + raise + except Exception as ex: + if "404" in str(ex): + print(f"[OK] Table removed: {table_name}") + else: + raise + + # ============================================================================ + # SUMMARY + # ============================================================================ + print("\n" + "=" * 80) + print("Async Walkthrough Complete!") + print("=" * 80) + print("\nDemonstrated operations:") + print(" [OK] Table creation with multiple column types") + print(" [OK] Single and multiple record creation") + print(" [OK] Reading records by ID and with filters") + print(" [OK] Single and multiple record updates") + print(" [OK] Paging through large result sets") + print(" [OK] AsyncQueryBuilder fluent queries (where + col(), col().in_(), col().between(), to_dataframe)") + print(" [OK] Expand navigation properties (simple + nested ExpandOption)") + print(" [OK] SQL queries") + print(" [OK] FetchXML queries (execute + execute_pages)") + print(" [OK] Picklist label-to-value conversion") + print(" [OK] Column management") + print(" [OK] Single and bulk delete operations") + print(" [OK] Batch operations (create, read, changeset, delete)") + print(" [OK] Table cleanup") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/basic/__init__.py b/examples/aio/basic/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/basic/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/basic/functional_testing.py b/examples/aio/basic/functional_testing.py new file mode 100644 index 00000000..b194050e --- /dev/null +++ b/examples/aio/basic/functional_testing.py @@ -0,0 +1,898 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client SDK - Async Functional Testing + +Async equivalent of examples/basic/functional_testing.py. + +This script provides comprehensive async functional testing of the SDK: +- Real environment connection testing +- Table creation and metadata operations +- Full CRUD operations testing +- Query functionality validation (list, list_pages, builder, fetchxml) +- Batch operations (create, read, update, changeset, delete) +- Interactive cleanup options + +Prerequisites: +- PowerPlatform-Dataverse-Client SDK installed (run aio/basic/installation_example.py first) +- Azure Identity credentials configured +- Access to a Dataverse environment with table creation permissions + +Usage: + python examples/aio/basic/functional_testing.py +""" + +import asyncio +import sys +from typing import Optional, Dict, Any +from datetime import datetime + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + + +def get_dataverse_org_url() -> str: + """Get Dataverse org URL from user input.""" + print("\n-> Dataverse Environment Setup") + print("=" * 50) + + if not sys.stdin.isatty(): + print("[ERR] Interactive input required. Run this script in a terminal.") + sys.exit(1) + + while True: + org_url = input("Enter your Dataverse org URL (e.g., https://yourorg.crm.dynamics.com): ").strip() + if org_url: + return org_url.rstrip("/") + print("[WARN] Please enter a valid URL.") + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off for metadata propagation delays.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" * Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print(f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total.") + raise last + + +async def setup_authentication(): + """Set up authentication and create async Dataverse client.""" + print("\n-> Authentication Setup") + print("=" * 50) + + org_url = get_dataverse_org_url() + try: + credential = AsyncInteractiveBrowserCredential() + client = AsyncDataverseClient(org_url, credential) + + print("Testing connection...") + tables = await client.tables.list() + print(f"[OK] Connection successful! Found {len(tables)} tables.") + + user_owned = await client.tables.list( + filter="OwnershipType eq Microsoft.Dynamics.CRM.OwnershipTypes'UserOwned'", + select=["LogicalName", "SchemaName", "DisplayName"], + ) + print(f"[OK] Found {len(user_owned)} user-owned tables (filter + select).") + return client, credential + + except Exception as e: + print(f"[ERR] Authentication failed: {e}") + sys.exit(1) + + +async def wait_for_table_metadata( + client: AsyncDataverseClient, + table_schema_name: str, + retries: int = 10, + delay_seconds: int = 3, +) -> Dict[str, Any]: + """Poll until table metadata is published and entity set becomes available.""" + for attempt in range(1, retries + 1): + try: + info = await client.tables.get(table_schema_name) + if info and info.get("entity_set_name"): + if attempt > 1: + print(f" [OK] Table metadata available after {attempt} attempts.") + return info + except Exception: + pass + + if attempt < retries: + print(f" Waiting for table metadata to publish (attempt {attempt}/{retries})...") + await asyncio.sleep(delay_seconds) + + raise RuntimeError("Table metadata did not become available in time. Please retry later.") + + +async def ensure_test_table(client: AsyncDataverseClient) -> Dict[str, Any]: + """Create or verify test table exists.""" + print("\n-> Test Table Setup") + print("=" * 50) + + table_schema_name = "test_TestSDKFunctionality" + + try: + existing_table = await client.tables.get(table_schema_name) + if existing_table: + print(f"[OK] Test table '{table_schema_name}' already exists") + return existing_table + except Exception: + print(f"Table '{table_schema_name}' not found, creating...") + + try: + print("Creating new test table...") + table_info = await client.tables.create( + table_schema_name, + primary_column="test_name", + columns={ + "test_description": "string", + "test_count": "int", + "test_amount": "decimal", + "test_is_active": "bool", + "test_created_date": "datetime", + }, + ) + print(f"[OK] Created test table: {table_info.get('table_schema_name')}") + print(f" Logical name: {table_info.get('table_logical_name')}") + print(f" Entity set: {table_info.get('entity_set_name')}") + + return await wait_for_table_metadata(client, table_schema_name) + + except MetadataError as e: + print(f"[ERR] Failed to create table: {e}") + sys.exit(1) + + +async def test_create_record(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> str: + """Test record creation.""" + print("\n-> Record Creation Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + test_data = { + f"{attr_prefix}_name": f"Test Record {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_description": "This is a test record created by the async SDK functionality test", + f"{attr_prefix}_count": 42, + f"{attr_prefix}_amount": 123.45, + f"{attr_prefix}_is_active": True, + f"{attr_prefix}_created_date": datetime.now().isoformat(), + } + + try: + print("Creating test record...") + created_id: Optional[str] = None + for attempt in range(1, retries + 1): + try: + created_id = await client.records.create(table_schema_name, test_data) + if attempt > 1: + print(f" [OK] Record creation succeeded after {attempt} attempts.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print( + f" Table not ready for create (attempt {attempt}/{retries}). Retrying in {delay_seconds}s..." + ) + await asyncio.sleep(delay_seconds) + continue + raise + + if created_id: + print(f"[OK] Record created successfully!") + print(f" Record ID: {created_id}") + return created_id + else: + raise ValueError("Unexpected response from records.create operation") + + except Exception as e: + print(f"[ERR] Failed to create record: {e}") + sys.exit(1) + + +async def test_read_record( + client: AsyncDataverseClient, + table_info: Dict[str, Any], + record_id: str, +) -> Dict[str, Any]: + """Test record reading.""" + print("\n-> Record Reading Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + try: + print(f"Reading record: {record_id}") + record = None + for attempt in range(1, retries + 1): + try: + record = await client.records.retrieve(table_schema_name, record_id) + if attempt > 1: + print(f" [OK] Record read succeeded after {attempt} attempts.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print(f" Record not queryable yet (attempt {attempt}/{retries}). Retrying in {delay_seconds}s...") + await asyncio.sleep(delay_seconds) + continue + raise + + if record is None: + raise RuntimeError("Record did not become available in time.") + + print("[OK] Record retrieved successfully!") + for field_name in [ + f"{attr_prefix}_name", + f"{attr_prefix}_description", + f"{attr_prefix}_count", + f"{attr_prefix}_amount", + f"{attr_prefix}_is_active", + ]: + if field_name in record: + print(f" {field_name}: {record[field_name]}") + + # include_annotations + annotation = "OData.Community.Display.V1.FormattedValue" + annotated = await client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_is_active", f"{attr_prefix}_count"], + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + if annotated is not None and ann_key in annotated: + print(f"[OK] include_annotations verified: {ann_key} = '{annotated[ann_key]}'") + else: + print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + + # expand + try: + expanded = await client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_name"], + expand=["owninguser"], + ) + owner = (expanded.get("owninguser") or {}) if expanded else {} + owner_name = owner.get("fullname") or owner.get("domainname") or "(unknown)" + print(f"[OK] records.retrieve with expand=['owninguser']: owner='{owner_name}'") + except Exception as e: + print(f"[WARN] records.retrieve expand skipped: {e}") + + return record + + except Exception as e: + print(f"[ERR] Failed to read record: {e}") + sys.exit(1) + + +async def test_query_records(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> None: + """Test querying multiple records.""" + print("\n-> Record Query Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + select_cols = [f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"] + active_filter = f"{attr_prefix}_is_active eq true" + + try: + # records.list() — eager + print("Querying records with await client.records.list()...") + for attempt in range(1, retries + 1): + try: + result = await client.records.list( + table_schema_name, + select=select_cols, + filter=active_filter, + top=5, + ) + record_count = 0 + for record in result: + record_count += 1 + name = record.get(f"{attr_prefix}_name", "N/A") + count = record.get(f"{attr_prefix}_count", "N/A") + amount = record.get(f"{attr_prefix}_amount", "N/A") + print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") + print(f"[OK] records.list() completed! Found {record_count} active records.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print(f" Query retry {attempt}/{retries}. Waiting {delay_seconds}s...") + await asyncio.sleep(delay_seconds) + continue + raise + + # records.list_pages() — lazy + print("\nQuerying records with async for page in client.records.list_pages() (paged)...") + page_num = 0 + total_records = 0 + async for page in client.records.list_pages( + table_schema_name, + select=select_cols, + filter=active_filter, + ): + page_num += 1 + total_records += len(page) + names = [r.get(f"{attr_prefix}_name", "N/A") for r in page] + print(f" Page {page_num}: {len(page)} record(s) — {names}") + print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + + # records.list() with extended params + print("\nQuerying records.list() with orderby / page_size / count / include_annotations...") + annotation = "OData.Community.Display.V1.FormattedValue" + annotated_result = await client.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in annotated_result) + if ann_present: + print(f"[OK] include_annotations verified: '{ann_key}' present in list() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found") + print(f"[OK] records.list() with extended params completed! {len(annotated_result)} record(s).") + + # AsyncQueryBuilder + from PowerPlatform.Dataverse.models.filters import col + + print("\nQuerying with AsyncQueryBuilder (.where(col(...)) + .page_size().execute_pages())...") + qb_pages = 0 + qb_total = 0 + async for page in ( + client.query.builder(table_schema_name) + .select(f"{attr_prefix}_name", f"{attr_prefix}_count") + .where(col(f"{attr_prefix}_is_active") == True) + .page_size(10) + .execute_pages() + ): + qb_pages += 1 + qb_total += len(page) + print(f"[OK] AsyncQueryBuilder execute_pages(): {qb_total} records across {qb_pages} page(s).") + + # FetchXML + print("\nQuerying with client.query.fetchxml().execute() ...") + fx_xml = f""" + + + + + + + + + + """ + try: + fx_result = await client.query.fetchxml(fx_xml).execute() + print(f"[OK] FetchXML execute(): {len(fx_result)} active records.") + except Exception as e: + print(f"[WARN] FetchXML query encountered an issue: {e}") + + except Exception as e: + print(f"[WARN] Query test encountered an issue: {e}") + print(" This might be expected if the table is very new.") + + +async def test_batch_all_operations(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> None: + """Test batch operations using the async batch client.""" + print("\n-> Batch Operations Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + logical_name = table_info.get("table_logical_name", table_schema_name.lower()) + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + all_ids: list = [] + + try: + # [1] CREATE — single + CreateMultiple + print("\n[1/7] Create — single + CreateMultiple") + batch = client.batch.new() + batch.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-A {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 1, + f"{attr_prefix}_is_active": True, + }, + ) + batch.records.create( + table_schema_name, + [ + { + f"{attr_prefix}_name": f"Batch-B {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 2, + f"{attr_prefix}_is_active": True, + }, + { + f"{attr_prefix}_name": f"Batch-C {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 3, + f"{attr_prefix}_is_active": True, + }, + ], + ) + result = await batch.execute() + all_ids = list(result.entity_ids) + if result.has_errors: + for item in result.failed: + print(f"[WARN] {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created") + + # [2] READ — retrieve + list + query.sql + if all_ids: + annotation = "OData.Community.Display.V1.FormattedValue" + print(f"\n[2/7] Read — records.retrieve + records.list + query.sql") + batch = client.batch.new() + batch.records.retrieve( + table_schema_name, + all_ids[0], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + include_annotations=annotation, + ) + batch.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=f"{attr_prefix}_is_active eq true", + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + include_annotations=annotation, + ) + batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") + result = await batch.execute() + print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") + + # [3] UPDATE — single + multiple + if len(all_ids) >= 2: + print(f"\n[3/7] Update — single PATCH + UpdateMultiple") + batch = client.batch.new() + batch.records.update(table_schema_name, all_ids[0], {f"{attr_prefix}_count": 10}) + batch.records.update(table_schema_name, all_ids[1:], {f"{attr_prefix}_count": 20}) + result = await batch.execute() + print(f"[OK] {len(result.succeeded)} updates succeeded") + + # [4] CHANGESET (happy path) — create + update via content-ID + delete + if all_ids: + print("\n[4/7] Changeset (happy path) — create + update(ref) + delete") + batch = client.batch.new() + async with batch.changeset() as cs: + ref = cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-D {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 4, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, ref, {f"{attr_prefix}_is_active": True}) + cs.records.delete(table_schema_name, all_ids[-1]) + result = await batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] {item.status_code}: {item.error_message}") + else: + new_id = next(iter(result.entity_ids), None) + if new_id: + all_ids[-1] = new_id + print(f"[OK] {len(result.succeeded)} ops committed atomically") + + # [5] CHANGESET (rollback) + print("\n[5/7] Changeset (rollback) — failing update rolls back create") + batch = client.batch.new() + async with batch.changeset() as cs: + cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Rollback-test {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 0, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, "00000000-0000-0000-0000-000000000001", {f"{attr_prefix}_count": 999}) + result = await batch.execute(continue_on_error=True) + if result.has_errors: + print("[OK] Changeset rollback verified: changeset failed, no records created") + else: + print("[WARN] Expected rollback but changeset succeeded (unexpected)") + all_ids.extend(result.entity_ids) + + # [6] ADD/REMOVE COLUMNS + col_a = f"{attr_prefix}_batch_extra_a" + col_b = f"{attr_prefix}_batch_extra_b" + print(f"\n[6/7] Batch tables.add_columns + tables.remove_columns") + batch = client.batch.new() + batch.tables.add_columns(table_schema_name, {col_a: "string"}) + batch.tables.add_columns(table_schema_name, {col_b: "int"}) + result = await batch.execute() + if not result.has_errors: + print(f"[OK] {len(result.succeeded)} column(s) added: {col_a}, {col_b}") + batch_rm = client.batch.new() + batch_rm.tables.remove_columns(table_schema_name, [col_a, col_b]) + rm_result = await batch_rm.execute(continue_on_error=True) + print(f"[OK] Removed {len(rm_result.succeeded)} batch-added column(s)") + else: + for item in result.failed: + print(f"[WARN] add_columns error {item.status_code}: {item.error_message}") + + # [7] DELETE + if all_ids: + print(f"\n[7/7] Delete — {len(all_ids)} records (use_bulk_delete=False)") + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + result = await batch.execute(continue_on_error=True) + print(f"[OK] Deleted {len(result.succeeded)}, failed {len(result.failed)}") + + print("\n[OK] Batch all-operations test completed!") + + except Exception as e: + print(f"[WARN] Batch test encountered an issue: {e}") + if all_ids: + try: + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + await batch.execute(continue_on_error=True) + except Exception: + pass + + +async def test_relationships(client: AsyncDataverseClient) -> None: + """Test relationship lifecycle: create tables, 1:N, N:N, query, delete.""" + print("\n-> Relationship Tests") + print("=" * 50) + + rel_parent_schema = "test_RelParent" + rel_child_schema = "test_RelChild" + rel_m2m_schema = "test_RelProject" + + rel_id_1n = None + rel_id_lookup = None + rel_id_nn = None + created_tables = [] + + try: + # Cleanup leftovers + print("Checking for leftover relationship test resources...") + found_leftovers = False + for rel_name in ["test_RelParent_RelChild", "contact_test_relchild_test_ManagerId", "test_relchild_relproject"]: + try: + rel = await client.tables.get_relationship(rel_name) + if rel: + found_leftovers = True + break + except Exception: + pass + + if not found_leftovers: + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if await client.tables.get(tbl): + found_leftovers = True + break + except Exception: + pass + + if found_leftovers: + cleanup_ok = input("Found leftover test resources. Clean up? (y/N): ").strip().lower() in ["y", "yes"] + if cleanup_ok: + for rel_name in [ + "test_RelParent_RelChild", + "contact_test_relchild_test_ManagerId", + "test_relchild_relproject", + ]: + try: + rel = await client.tables.get_relationship(rel_name) + if rel: + await client.tables.delete_relationship(rel.relationship_id) + print(f" (Cleaned up relationship: {rel_name})") + except Exception: + pass + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if await client.tables.get(tbl): + await client.tables.delete(tbl) + print(f" (Cleaned up table: {tbl})") + except Exception: + pass + + # Create tables + print("\nCreating relationship test tables...") + + async def _get_or_create(schema, columns, label): + info = await client.tables.get(schema) + if info: + print(f"[OK] Table already exists: {schema} (skipped)") + return info + try: + result = await backoff(lambda: client.tables.create(schema, columns)) + print(f"[OK] Created {label}: {schema}") + return result + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {schema} (skipped)") + return await client.tables.get(schema) + raise + + parent_info = await _get_or_create(rel_parent_schema, {"test_Code": "string"}, "parent table") + created_tables.append(rel_parent_schema) + + child_info = await _get_or_create(rel_child_schema, {"test_Number": "string"}, "child table") + created_tables.append(rel_child_schema) + + proj_info = await _get_or_create(rel_m2m_schema, {"test_ProjectCode": "string"}, "M:N table") + created_tables.append(rel_m2m_schema) + + await wait_for_table_metadata(client, rel_parent_schema) + await wait_for_table_metadata(client, rel_child_schema) + await wait_for_table_metadata(client, rel_m2m_schema) + + # 1:N relationship + print("\n Test 1: Create 1:N relationship") + lookup = LookupAttributeMetadata( + schema_name="test_ParentId", + display_name=Label(localized_labels=[LocalizedLabel(label="Parent", language_code=1033)]), + required_level="None", + ) + relationship = OneToManyRelationshipMetadata( + schema_name="test_RelParent_RelChild", + referenced_entity=parent_info["table_logical_name"], + referencing_entity=child_info["table_logical_name"], + referenced_attribute=f"{parent_info['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + merge=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + existing_1n = await client.tables.get_relationship("test_RelParent_RelChild") + if existing_1n: + rel_id_1n = existing_1n.relationship_id + print(f" [OK] Relationship already exists (skipped)") + else: + result_1n = await backoff( + lambda: client.tables.create_one_to_many_relationship(lookup=lookup, relationship=relationship) + ) + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N: {result_1n.relationship_schema_name}") + + # Lookup field + print("\n Test 2: Create lookup field (convenience API)") + existing_lookup = await client.tables.get_relationship("contact_test_relchild_test_ManagerId") + if existing_lookup: + rel_id_lookup = existing_lookup.relationship_id + print(f" [OK] Lookup already exists (skipped)") + else: + result_lookup = await backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + + # N:N relationship + print("\n Test 3: Create N:N relationship") + m2m = ManyToManyRelationshipMetadata( + schema_name="test_relchild_relproject", + entity1_logical_name=child_info["table_logical_name"], + entity2_logical_name=proj_info["table_logical_name"], + ) + existing_nn = await client.tables.get_relationship("test_relchild_relproject") + if existing_nn: + rel_id_nn = existing_nn.relationship_id + print(f" [OK] Relationship already exists (skipped)") + else: + result_nn = await backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + assert result_nn.relationship_schema_name == "test_relchild_relproject" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N: {result_nn.relationship_schema_name}") + + # Get relationship metadata + print("\n Test 4: Query relationship metadata") + fetched_1n = await client.tables.get_relationship("test_RelParent_RelChild") + assert fetched_1n is not None and fetched_1n.relationship_type == "one_to_many" + print(f" [OK] Retrieved 1:N: {fetched_1n.relationship_schema_name}") + + fetched_nn = await client.tables.get_relationship("test_relchild_relproject") + assert fetched_nn is not None and fetched_nn.relationship_type == "many_to_many" + print(f" [OK] Retrieved N:N: {fetched_nn.relationship_schema_name}") + + missing = await client.tables.get_relationship("nonexistent_relationship_xyz") + assert missing is None + print(" [OK] Non-existent relationship returns None") + + # Delete relationships + print("\n Test 5: Delete relationships") + await backoff(lambda: client.tables.delete_relationship(rel_id_1n)) + rel_id_1n = None + print(" [OK] Deleted 1:N relationship") + + await backoff(lambda: client.tables.delete_relationship(rel_id_lookup)) + rel_id_lookup = None + print(" [OK] Deleted lookup relationship") + + await backoff(lambda: client.tables.delete_relationship(rel_id_nn)) + rel_id_nn = None + print(" [OK] Deleted N:N relationship") + + verify = await client.tables.get_relationship("test_RelParent_RelChild") + assert verify is None + print(" [OK] Verified 1:N deletion") + + print("\n[OK] All relationship tests passed!") + + finally: + for rid in [rel_id_1n, rel_id_lookup, rel_id_nn]: + if rid: + try: + await client.tables.delete_relationship(rid) + except Exception: + pass + + for tbl in reversed(created_tables): + try: + await backoff(lambda name=tbl: client.tables.delete(name)) + print(f" (Cleaned up table: {tbl})") + except Exception as e: + print(f" [WARN] Could not delete {tbl}: {e}") + + +async def cleanup_test_data( + client: AsyncDataverseClient, + table_info: Dict[str, Any], + record_id: str, +) -> None: + """Clean up test data.""" + print("\n-> Cleanup") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + retries = 5 + delay_seconds = 3 + + cleanup_choice = input("Do you want to delete the test record? (y/N): ").strip().lower() + if cleanup_choice in ["y", "yes"]: + for attempt in range(1, retries + 1): + try: + await client.records.delete(table_schema_name, record_id) + print("[OK] Test record deleted successfully") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404: + print("Record already deleted; skipping.") + break + if attempt < retries: + await asyncio.sleep(delay_seconds) + continue + print(f"[WARN] Failed to delete test record: {err}") + except Exception as e: + print(f"[WARN] Failed to delete test record: {e}") + break + else: + print("Test record kept for inspection") + + table_cleanup = input("Do you want to delete the test table? (y/N): ").strip().lower() + if table_cleanup in ["y", "yes"]: + for attempt in range(1, retries + 1): + try: + await client.tables.delete(table_schema_name) + print("[OK] Test table deleted successfully") + break + except HttpError as err: + if attempt < retries: + await asyncio.sleep(delay_seconds) + continue + print(f"[WARN] Failed to delete test table: {err}") + except Exception as e: + print(f"[WARN] Failed to delete test table: {e}") + break + else: + print("Test table kept for future testing") + + +async def main(): + """Main async test function.""" + print("PowerPlatform Dataverse Client SDK - Async Functional Testing") + print("=" * 70) + print("This script tests async SDK functionality in a real Dataverse environment:") + print(" - Authentication & Connection") + print(" - Table Creation & Metadata Operations") + print(" - Record CRUD Operations") + print(" - Query Functionality (list, list_pages, builder, fetchxml)") + print(" - Relationship Operations (1:N, N:N, lookup)") + print(" - Batch Operations (create, read, update, changeset, delete)") + print(" - Interactive Cleanup") + print("=" * 70) + print("For installation validation, run examples/aio/basic/installation_example.py first") + print("=" * 70) + + try: + client, credential = await setup_authentication() + + try: + async with client: + table_info = await ensure_test_table(client) + record_id = await test_create_record(client, table_info) + await test_read_record(client, table_info, record_id) + await test_query_records(client, table_info) + await test_relationships(client) + await test_batch_all_operations(client, table_info) + + print("\nAsync Functional Test Summary") + print("=" * 50) + print("[OK] Authentication: Success") + print("[OK] Table Operations: Success") + print("[OK] Record Creation: Success") + print("[OK] Record Reading: Success") + print("[OK] Record Querying (list, list_pages, builder, fetchxml): Success") + print("[OK] Relationship Operations: Success") + print("[OK] Batch Operations: Success") + print("\nYour async PowerPlatform Dataverse Client SDK is fully functional!") + + await cleanup_test_data(client, table_info, record_id) + finally: + await credential.close() + + except KeyboardInterrupt: + print("\n\n[WARN] Test interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n[ERR] Unexpected error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/basic/installation_example.py b/examples/aio/basic/installation_example.py new file mode 100644 index 00000000..df958c17 --- /dev/null +++ b/examples/aio/basic/installation_example.py @@ -0,0 +1,372 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Installation, Validation & Usage Example + +Async equivalent of examples/basic/installation_example.py. + +This script demonstrates the async client (AsyncDataverseClient) and validates +that all async imports, classes, and methods are correctly installed. + +## Installation + +```bash +pip install PowerPlatform-Dataverse-Client azure-identity +``` + +## What This Script Does + +- Validates async package imports +- Checks version and package metadata +- Shows async usage patterns +- Offers optional interactive testing with a real Dataverse environment + +Prerequisites for Interactive Testing: +- Access to a Microsoft Dataverse environment +- Azure Identity credentials configured +- Interactive browser access for authentication +""" + +import asyncio +import sys +import subprocess +from datetime import datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations + + +def validate_imports(): + """Validate that all key async imports work correctly.""" + print("Validating Async Package Imports...") + print("-" * 50) + + try: + from PowerPlatform.Dataverse import __version__ + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + print(f" [OK] Namespace: PowerPlatform.Dataverse.aio") + print(f" [OK] Package version: {__version__}") + print(f" [OK] Async client: PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient") + + from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError + + print(f" [OK] Core errors: HttpError, MetadataError") + + from PowerPlatform.Dataverse.core.config import DataverseConfig + + print(f" [OK] Core config: DataverseConfig") + + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + print(f" [OK] Async data layer: _AsyncODataClient") + + from PowerPlatform.Dataverse.aio.models.async_fetchxml_query import AsyncFetchXmlQuery + from PowerPlatform.Dataverse.aio.models.async_query_builder import AsyncQueryBuilder + + print(f" [OK] Async models: AsyncFetchXmlQuery, AsyncQueryBuilder") + + from _auth import AsyncInteractiveBrowserCredential + + print(f" [OK] Azure Identity: AsyncInteractiveBrowserCredential (interactive browser)") + + return True, __version__, AsyncDataverseClient + + except ImportError as e: + print(f" [ERR] Import failed: {e}") + print("\nTroubleshooting:") + print(" pip install PowerPlatform-Dataverse-Client azure-identity") + print(" Or for development: pip install -e .") + return False, None, None + + +def validate_client_methods(AsyncDataverseClient): + """Validate that AsyncDataverseClient has expected methods.""" + print("\nValidating Async Client Methods...") + print("-" * 50) + + expected_namespaces = { + "records": ["create", "retrieve", "update", "delete", "list", "list_pages", "upsert"], + "query": ["sql", "builder", "fetchxml", "sql_columns", "odata_expands"], + "tables": [ + "create", + "get", + "list", + "delete", + "add_columns", + "remove_columns", + "create_one_to_many_relationship", + "create_many_to_many_relationship", + "delete_relationship", + "get_relationship", + "create_lookup_field", + ], + "files": ["upload"], + } + + ns_classes = { + "records": AsyncRecordOperations, + "query": AsyncQueryOperations, + "tables": AsyncTableOperations, + "files": AsyncFileOperations, + } + + missing_methods = [] + for ns, methods in expected_namespaces.items(): + ns_cls = ns_classes.get(ns) + for method in methods: + attr_path = f"{ns}.{method}" + if ns_cls is not None and hasattr(ns_cls, method): + print(f" [OK] Method exists: {attr_path}") + else: + print(f" [ERR] Method missing: {attr_path}") + missing_methods.append(attr_path) + + return len(missing_methods) == 0 + + +def validate_package_metadata(): + """Validate package metadata from pip.""" + print("\nValidating Package Metadata...") + print("-" * 50) + + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "show", "PowerPlatform-Dataverse-Client"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + for line in result.stdout.split("\n"): + if any(line.startswith(p) for p in ["Name:", "Version:", "Summary:", "Location:"]): + print(f" [OK] {line}") + return True + else: + print(" [ERR] Package not found in pip list") + return False + except Exception as e: + print(f" [ERR] Metadata validation failed: {e}") + return False + + +def show_usage_examples(): + """Display async usage examples.""" + print("\nAsync Usage Examples") + print("=" * 50) + + print(""" +Basic Setup: +```python +import asyncio +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + +async def main(): + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient("https://yourorg.crm.dynamics.com", credential) as client: + ... # all operations here + finally: + await credential.close() + +asyncio.run(main()) +``` + +CRUD Operations: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + # Create a record + account_id = await client.records.create("account", {"name": "Contoso Ltd"}) + + # Read a single record by ID + account = await client.records.retrieve("account", account_id) + print(f"Account name: {account['name']}") + + # Update a record + await client.records.update("account", account_id, {"telephone1": "555-0200"}) + + # Delete a record + await client.records.delete("account", account_id) +``` + +Querying Data: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + from PowerPlatform.Dataverse.models.filters import col + + # Fluent query builder + result = await ( + client.query.builder("account") + .select("name", "telephone1") + .where(col("statecode") == 0) + .top(10) + .execute() + ) + for record in result: + print(record["name"]) + + # Lazy paged iteration + async for page in ( + client.query.builder("account") + .select("name") + .page_size(50) + .execute_pages() + ): + for record in page: + print(record["name"]) + + # SQL query + rows = await client.query.sql("SELECT TOP 5 name FROM account") + for row in rows: + print(row["name"]) + + # FetchXML + xml = '' + rows = await client.query.fetchxml(xml).execute() + for row in rows: + print(row["name"]) +``` + +Batch Operations: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + batch = client.batch.new() + batch.records.create("account", {"name": "Alpha"}) + batch.records.create("account", {"name": "Beta"}) + result = await batch.execute() + print(f"Created {len(list(result.entity_ids))} records") + + # Atomic changeset + batch = client.batch.new() + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + result = await batch.execute() +``` +""") + + +async def interactive_test(): + """Offer optional interactive testing with real Dataverse environment.""" + print("\nInteractive Testing") + print("=" * 50) + + choice = input("Would you like to test with a real Dataverse environment? (y/N): ").strip().lower() + if choice not in ["y", "yes"]: + print(" Skipping interactive test") + return + + if not sys.stdin.isatty(): + print(" [ERR] Interactive input required for testing") + return + + org_url = input("Enter your Dataverse org URL (e.g., https://yourorg.crm.dynamics.com): ").strip() + if not org_url: + print(" [WARN] No URL provided, skipping test") + return + + try: + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + from _auth import AsyncInteractiveBrowserCredential + + print(" Setting up authentication...") + credential = AsyncInteractiveBrowserCredential() + + print(" Creating async client...") + try: + async with AsyncDataverseClient(org_url.rstrip("/"), credential) as client: + print(" Testing connection...") + tables = await client.tables.list() + print(f" [OK] Connection successful!") + print(f" Found {len(tables)} tables in environment") + + custom_tables = await client.tables.list( + filter="IsCustomEntity eq true", + select=["LogicalName", "SchemaName"], + ) + print(f" Found {len(custom_tables)} custom tables (filter + select)") + finally: + await credential.close() + + print("\n Your async SDK is ready for use!") + + except Exception as e: + print(f" [ERR] Interactive test failed: {e}") + print(" This might be due to authentication, network, or permissions") + print(" The SDK imports are still valid for offline development") + + +async def main(): + """Run async installation validation and demonstration.""" + print("PowerPlatform Dataverse Client SDK - Async Installation & Validation") + print("=" * 70) + print(f"Validation Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 70) + + imports_success, version, AsyncDataverseClient = validate_imports() + if not imports_success: + print("\n[ERR] Import validation failed. Please check installation.") + sys.exit(1) + + methods_success = True + if AsyncDataverseClient: + methods_success = validate_client_methods(AsyncDataverseClient) + if not methods_success: + print("\n[WARN] Some client methods are missing, but basic functionality should work.") + + metadata_success = validate_package_metadata() + + show_usage_examples() + + await interactive_test() + + print("\n" + "=" * 70) + print("VALIDATION SUMMARY") + print("=" * 70) + + results = [ + ("Async Package Imports", imports_success), + ("Async Client Methods", methods_success), + ("Package Metadata", metadata_success), + ] + + all_passed = True + for test_name, success in results: + status = "[OK] PASS" if success else "[ERR] FAIL" + print(f"{test_name:<25} {status}") + if not success: + all_passed = False + + print("=" * 70) + if all_passed: + print("SUCCESS: Async PowerPlatform-Dataverse-Client is properly installed!") + if version: + print(f"Package Version: {version}") + print("\nNext Steps:") + print(" - Run examples/aio/basic/functional_testing.py for a live test") + print(" - Run examples/aio/advanced/walkthrough.py for a full feature tour") + else: + print("[ERR] Some validation checks failed!") + print(" pip uninstall PowerPlatform-Dataverse-Client") + print(" pip install PowerPlatform-Dataverse-Client") + sys.exit(1) + + +if __name__ == "__main__": + print("PowerPlatform-Dataverse-Client SDK - Async Installation Example") + print("=" * 60) + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 77a6d492..1ddcc51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,27 +40,30 @@ dependencies = [ [project.scripts] dataverse-install-claude-skill = "PowerPlatform.Dataverse._skill_installer:main" -dataverse-migrate = "PowerPlatform.Dataverse.migration.migrate_v0_to_v1:main" +dataverse-migrate = "tools.migrate_v0_to_v1:main" [project.optional-dependencies] +async = [ + "aiohttp>=3.9", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", + "pytest-asyncio>=0.23.0", "black>=23.0.0", "isort>=5.12.0", "mypy>=1.0.0", "ruff>=0.1.0", - "libcst>=1.0.0", ] migration = ["libcst>=1.0.0"] [tool.setuptools] -package-dir = {"" = "src"} +package-dir = {"" = "src", "tools" = "tools"} zip-safe = false [tool.setuptools.packages.find] -where = ["src"] -include = ["PowerPlatform*"] +where = ["src", "."] +include = ["PowerPlatform*", "tools"] namespaces = false [tool.setuptools.package-data] @@ -96,15 +99,14 @@ select = [ [tool.pytest.ini_options] testpaths = ["tests/unit"] +asyncio_mode = "auto" [tool.coverage.run] source = ["src/PowerPlatform"] -# Migration codemod is an opt-in tool (requires the [migration] extra). -# It is exercised by tests/unit/test_migration_tool.py and tests/unit/test_phase4_ga.py, -# but its CLI plumbing and libcst error-path branches are not meaningfully testable -# under the default coverage scope. Exclude from coverage reporting so the metric -# reflects core SDK code only. -omit = ["src/PowerPlatform/Dataverse/migration/*"] +omit = [ + "*/Dataverse/_skill_installer.py", + "*/Dataverse/extensions/__init__.py", +] [tool.coverage.report] fail_under = 90 diff --git a/src/PowerPlatform/Dataverse/aio/__init__.py b/src/PowerPlatform/Dataverse/aio/__init__.py new file mode 100644 index 00000000..ab39e8d8 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async namespace for the PowerPlatform Dataverse SDK. + +Import the async client via:: + + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +""" + +__all__ = [] diff --git a/src/PowerPlatform/Dataverse/aio/async_client.py b/src/PowerPlatform/Dataverse/aio/async_client.py new file mode 100644 index 00000000..30553883 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/async_client.py @@ -0,0 +1,252 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import AsyncIterator, Optional + +import aiohttp +from azure.core.credentials_async import AsyncTokenCredential + +from .core._async_auth import _AsyncAuthManager +from ..core.config import DataverseConfig, OperationContext +from .data._async_odata import _AsyncODataClient +from .operations.async_dataframe import AsyncDataFrameOperations +from .operations.async_records import AsyncRecordOperations +from .operations.async_query import AsyncQueryOperations +from .operations.async_files import AsyncFileOperations +from .operations.async_tables import AsyncTableOperations +from .operations.async_batch import AsyncBatchOperations + + +class AsyncDataverseClient: + """ + Async high-level client for Microsoft Dataverse operations. + + This client provides a simple, stable async interface for interacting with + Dataverse environments through the Web API. It handles authentication via + Azure Identity and delegates HTTP operations to an internal + :class:`~PowerPlatform.Dataverse.aio.data._async_odata._AsyncODataClient`. + + Key capabilities: + - OData CRUD operations: create, read, update, delete records + - SQL queries: execute read-only SQL via Web API ``?sql`` parameter + - Table metadata: create, inspect, and delete custom tables; create and delete columns + - File uploads: upload files to file columns with chunking support + + :param base_url: Your Dataverse environment URL, for example + ``"https://org.crm.dynamics.com"``. Trailing slash is automatically removed. + :type base_url: :class:`str` + :param credential: Azure async Identity credential for authentication. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential + :param config: Optional configuration for language, timeouts, and retries. + If not provided, defaults are loaded from :meth:`~PowerPlatform.Dataverse.core.config.DataverseConfig.from_env`. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig or None + :param context: Optional caller-defined context object appended to the + outbound ``User-Agent`` header for plugin/tool attribution. Cannot be used + together with ``config`` -- pass the context via + :class:`~PowerPlatform.Dataverse.core.config.DataverseConfig` instead. + :type context: ~PowerPlatform.Dataverse.core.config.OperationContext or None + + :raises ValueError: If ``base_url`` is missing or empty after trimming. + :raises ValueError: If both ``config`` and ``context`` are provided. + + .. note:: + The client lazily initializes its internal OData client on first use, + allowing lightweight construction without immediate network calls. + + .. note:: + All methods that communicate with the Dataverse Web API may raise + :class:`~PowerPlatform.Dataverse.core.errors.HttpError` on non-successful + HTTP responses (e.g. 401, 403, 404, 429, 500). Individual method + docstrings document only domain-specific exceptions. + + Operations are organized into namespaces: + + - ``client.records`` -- create, update, delete, and get records (single or paginated queries) + - ``client.query`` -- query and search operations + - ``client.tables`` -- table and column metadata management + - ``client.files`` -- file upload operations + - ``client.dataframe`` -- pandas DataFrame wrappers for record CRUD + - ``client.batch`` -- batch multiple operations into a single HTTP request + + The client supports Python's async context manager protocol for automatic + resource cleanup and HTTP connection pooling: + + Example: + **Recommended -- async context manager** (enables HTTP connection pooling):: + + from azure.identity.aio import InteractiveBrowserCredential + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + credential = InteractiveBrowserCredential() + + async with AsyncDataverseClient("https://org.crm.dynamics.com", credential) as client: + record_id = await client.records.create("account", {"name": "Contoso Ltd"}) + await client.records.update("account", record_id, {"telephone1": "555-0100"}) + # Session closed, caches cleared automatically + + **Manual lifecycle**:: + + client = AsyncDataverseClient("https://org.crm.dynamics.com", credential) + try: + record_id = await client.records.create("account", {"name": "Contoso Ltd"}) + finally: + await client.aclose() + """ + + def __init__( + self, + base_url: str, + credential: AsyncTokenCredential, + config: Optional[DataverseConfig] = None, + *, + context: Optional[OperationContext] = None, + ) -> None: + if config is not None and context is not None: + raise ValueError( + "Cannot specify both 'config' and 'context'. " "Pass operation_context via DataverseConfig instead." + ) + self.auth = _AsyncAuthManager(credential) + self._base_url = (base_url or "").rstrip("/") + if not self._base_url: + raise ValueError("base_url is required.") + if config is not None: + self._config = config + elif context is not None: + self._config = DataverseConfig(operation_context=context) + else: + self._config = DataverseConfig.from_env() + self._odata: Optional[_AsyncODataClient] = None + self._session: Optional[aiohttp.ClientSession] = None + self._closed: bool = False + + # Operation namespaces + self.records = AsyncRecordOperations(self) + self.query = AsyncQueryOperations(self) + self.tables = AsyncTableOperations(self) + self.files = AsyncFileOperations(self) + self.dataframe = AsyncDataFrameOperations(self) + self.batch = AsyncBatchOperations(self) + + def _get_odata(self) -> _AsyncODataClient: + """ + Get or create the internal async OData client instance. + + This method implements lazy initialization of the low-level async OData + client, deferring construction until the first API call. When used outside + of an ``async with`` block, a :class:`aiohttp.ClientSession` is created + lazily here so that standalone usage (without a context manager) works + without requiring the caller to manage the session explicitly. + + :return: The lazily-initialized low-level async client. + :rtype: ~PowerPlatform.Dataverse.aio.data._async_odata._AsyncODataClient + """ + if self._odata is None: + if self._session is None: + self._session = aiohttp.ClientSession() + self._odata = _AsyncODataClient( + self.auth, + self._base_url, + self._config, + session=self._session, + ) + return self._odata + + @asynccontextmanager + async def _scoped_odata(self) -> AsyncIterator[_AsyncODataClient]: + """Async context manager yielding the low-level client with a correlation scope.""" + self._check_closed() + od = self._get_odata() + # _call_scope() is a sync context manager (just sets a context var — no I/O). + with od._call_scope(): + yield od + + # ---------------- Context manager / lifecycle ---------------- + + async def __aenter__(self) -> "AsyncDataverseClient": + """Enter the async context manager. + + Creates an :class:`aiohttp.ClientSession` for HTTP connection pooling. + All operations within the ``async with`` block reuse this session for + better performance (TCP and TLS reuse). + + :return: The client instance. + :rtype: AsyncDataverseClient + + :raises RuntimeError: If the client has been closed. + """ + self._check_closed() + if self._session is None: + self._session = aiohttp.ClientSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + """Exit the async context manager with cleanup. + + Calls :meth:`aclose` to release resources. Exceptions are not + suppressed. + """ + await self.aclose() + + async def aclose(self) -> None: + """Close the async client and release resources. + + Closes the HTTP session (if any), clears internal caches, and + marks the client as closed. Safe to call multiple times. After + closing, any operation will raise :class:`RuntimeError`. + + Called automatically when using the client as an async context manager. + + Example:: + + client = AsyncDataverseClient(base_url, credential) + try: + await client.records.create("account", {"name": "Contoso"}) + finally: + await client.aclose() + """ + if self._closed: + return + if self._odata is not None: + await self._odata.close() + self._odata = None + if self._session is not None: + await self._session.close() + self._session = None + self._closed = True + + def _check_closed(self) -> None: + """Raise :class:`RuntimeError` if the client has been closed.""" + if self._closed: + raise RuntimeError("AsyncDataverseClient is closed") + + # ---------------- Cache utilities ---------------- + + async def flush_cache(self, kind: str) -> int: + """ + Flush cached client metadata or state. + + :param kind: Cache kind to flush. Currently supported values: + + - ``"picklist"``: Clears picklist label cache used for label-to-integer conversion + + Future kinds (e.g. ``"entityset"``, ``"primaryid"``) may be added without + breaking this signature. + :type kind: :class:`str` + + :return: Number of cache entries removed. + :rtype: :class:`int` + + Example: + Clear the picklist cache:: + + removed = await client.flush_cache("picklist") + print(f"Cleared {removed} cached picklist entries") + """ + async with self._scoped_odata() as od: + return od._flush_cache(kind) + + +__all__ = ["AsyncDataverseClient"] diff --git a/src/PowerPlatform/Dataverse/aio/core/__init__.py b/src/PowerPlatform/Dataverse/aio/core/__init__.py new file mode 100644 index 00000000..2f0cb105 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async core infrastructure components for the Dataverse SDK. + +This module contains the foundational async components including authentication, +HTTP client, and error handling. +""" diff --git a/src/PowerPlatform/Dataverse/aio/core/_async_auth.py b/src/PowerPlatform/Dataverse/aio/core/_async_auth.py new file mode 100644 index 00000000..ca25d5b1 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/_async_auth.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async authentication helpers for Dataverse. + +This module provides :class:`~PowerPlatform.Dataverse.aio.core._async_auth._AsyncAuthManager`, +a thin wrapper over any Azure Identity ``AsyncTokenCredential`` for acquiring OAuth2 access +tokens asynchronously, and reuses :class:`~PowerPlatform.Dataverse.core._auth._TokenPair` for +storing the acquired token alongside its scope. +""" + +from __future__ import annotations + +from azure.core.credentials_async import AsyncTokenCredential + +from ...core._auth import _TokenPair + + +class _AsyncAuthManager: + """ + Azure Identity-based async authentication manager for Dataverse. + + :param credential: Azure Identity async credential implementation. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential + :raises TypeError: If ``credential`` does not implement :class:`~azure.core.credentials_async.AsyncTokenCredential`. + """ + + def __init__(self, credential: AsyncTokenCredential) -> None: + if not isinstance(credential, AsyncTokenCredential): + raise TypeError("credential must implement azure.core.credentials_async.AsyncTokenCredential.") + self.credential: AsyncTokenCredential = credential + + async def _acquire_token(self, scope: str) -> _TokenPair: + """ + Acquire an access token asynchronously for the specified OAuth2 scope. + + :param scope: OAuth2 scope string, typically ``"https://.crm.dynamics.com/.default"``. + :type scope: :class:`str` + :return: Token pair containing the scope and access token. + :rtype: ~PowerPlatform.Dataverse.core._auth._TokenPair + :raises ~azure.core.exceptions.ClientAuthenticationError: If token acquisition fails. + """ + token = await self.credential.get_token(scope) + return _TokenPair(resource=scope, access_token=token.token) diff --git a/src/PowerPlatform/Dataverse/aio/core/_async_http.py b/src/PowerPlatform/Dataverse/aio/core/_async_http.py new file mode 100644 index 00000000..402be8d4 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/_async_http.py @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async HTTP client with automatic retry logic and timeout handling. + +This module provides :class:`~PowerPlatform.Dataverse.aio.core._async_http._AsyncHttpClient`, +a wrapper around the aiohttp library that adds configurable retry behavior for transient +network errors and intelligent timeout management based on HTTP method types. +""" + +from __future__ import annotations + +import asyncio +import json as _json +import time +from typing import TYPE_CHECKING, Any, Dict, Optional + +import aiohttp + +if TYPE_CHECKING: + from ...core._http_logger import _HttpLogger + + +class _AsyncResponse: + """Materialized HTTP response returned by :class:`_AsyncHttpClient._request`. + + The body is fully buffered before this object is constructed, so all + accessors are synchronous — no ``await`` required. + + :param status: HTTP status code. + :param headers: Response headers as a plain dict. + :param body: Raw response body bytes. + """ + + __slots__ = ("status", "status_code", "headers", "_body") + + def __init__(self, status: int, headers: Dict[str, str], body: bytes) -> None: + self.status = status + self.status_code = status + self.headers = headers + self._body = body + + @property + def text(self) -> str: + """Response body decoded as UTF-8 text.""" + return self._body.decode("utf-8", errors="replace") if self._body else "" + + def json(self, content_type: Any = None) -> Any: + """Parse and return the response body as JSON.""" + return _json.loads(self._body) if self._body else {} + + +class _AsyncHttpClient: + """ + Async HTTP client with configurable retry logic and timeout handling. + + Provides automatic retry behavior for transient failures and default timeout + management for different HTTP methods. + + :param retries: Maximum number of retry attempts for transient errors. Default is 5. + :type retries: :class:`int` | None + :param backoff: Base delay in seconds between retry attempts. Default is 0.5. + :type backoff: :class:`float` | None + :param timeout: Default request timeout in seconds. If None, uses per-method defaults. + :type timeout: :class:`float` | None + :param session: ``aiohttp.ClientSession`` for HTTP connection pooling. + The session is owned by the caller (``AsyncDataverseClient``) and must remain + open for the lifetime of this client. Unlike the sync client, there is no + per-request fallback — a session must always be provided before making requests. + :type session: :class:`aiohttp.ClientSession` | None + :param logger: Optional HTTP diagnostics logger. When provided, all requests, + responses, and transport errors are logged with automatic header redaction. + :type logger: ~PowerPlatform.Dataverse.core._http_logger._HttpLogger | None + """ + + def __init__( + self, + retries: Optional[int] = None, + backoff: Optional[float] = None, + timeout: Optional[float] = None, + session: Optional[aiohttp.ClientSession] = None, + logger: Optional["_HttpLogger"] = None, + ) -> None: + self.max_attempts = retries if retries is not None else 5 + self.base_delay = backoff if backoff is not None else 0.5 + self.default_timeout: Optional[float] = timeout + self._session = session + self._logger = logger + + async def _request(self, method: str, url: str, **kwargs: Any) -> _AsyncResponse: + """ + Execute an HTTP request asynchronously with automatic retry logic and timeout management. + + Applies default timeouts based on HTTP method (120s for POST/DELETE, 10s for others) + and retries on network errors with exponential backoff. + + The response body is fully buffered and returned as a :class:`_AsyncResponse` whose + accessors (``.text``, ``.json()``) are synchronous — no ``await`` required on the caller side. + + :param method: HTTP method (GET, POST, PUT, DELETE, etc.). + :type method: :class:`str` + :param url: Target URL for the request. + :type url: :class:`str` + :param kwargs: Additional arguments passed to ``aiohttp.ClientSession.request()``, + including headers, data, etc. + :return: Materialized HTTP response with body fully buffered. + :rtype: :class:`_AsyncResponse` + :raises aiohttp.ClientError: If all retry attempts fail. + :raises RuntimeError: If no session has been set. + """ + if self._session is None: + raise RuntimeError("No aiohttp.ClientSession set. Set _session before making requests.") + + # If no timeout is provided, use the user-specified default timeout if set; + # otherwise, apply per-method defaults (120s for POST/DELETE, 10s for others). + if "timeout" not in kwargs: + if self.default_timeout is not None: + t = self.default_timeout + else: + m = (method or "").lower() + t = 120 if m in ("post", "delete") else 10 + kwargs["timeout"] = aiohttp.ClientTimeout(total=t) + + # Log outbound request once (before retry loop). + # Use explicit key presence checks so falsy values (e.g. {}) are logged correctly. + if self._logger is not None: + if "json" in kwargs: + req_body = kwargs["json"] + elif "data" in kwargs: + req_body = kwargs["data"] + else: + req_body = None + self._logger.log_request( + method, + url, + headers=kwargs.get("headers"), + body=req_body, + ) + + # Small backoff retry on network errors only + for attempt in range(self.max_attempts): + try: + t0 = time.monotonic() + async with self._session.request(method, url, **kwargs) as resp: + body = await resp.read() + materialized = _AsyncResponse(resp.status, dict(resp.headers), body) + elapsed_ms = (time.monotonic() - t0) * 1000 + + if self._logger is not None: + # Only decode text when body logging is enabled — avoids + # unnecessary overhead for large payloads when max_body_bytes == 0. + resp_body = materialized.text if self._logger.body_logging_enabled else None + self._logger.log_response( + method, + url, + status_code=materialized.status, + headers=materialized.headers, + body=resp_body, + elapsed_ms=elapsed_ms, + ) + return materialized + except (aiohttp.ClientError, asyncio.TimeoutError) as exc: + if self._logger is not None: + self._logger.log_error( + method, + url, + exc, + attempt=attempt + 1, + max_attempts=self.max_attempts, + ) + if attempt == self.max_attempts - 1: + raise + delay = self.base_delay * (2**attempt) + await asyncio.sleep(delay) + continue + + async def close(self) -> None: + """Close the HTTP client and release resources. + + If a session was provided, closes it. Safe to call multiple times. + """ + if self._session is not None: + await self._session.close() + self._session = None diff --git a/src/PowerPlatform/Dataverse/aio/data/__init__.py b/src/PowerPlatform/Dataverse/aio/data/__init__.py new file mode 100644 index 00000000..be8f223c --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async data access layer for the Dataverse SDK. + +This module contains async OData protocol handling, CRUD operations, metadata management, +SQL query functionality, and file upload capabilities. +""" diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_batch.py b/src/PowerPlatform/Dataverse/aio/data/_async_batch.py new file mode 100644 index 00000000..967929dd --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_batch.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async batch intent resolver and dispatcher for the Dataverse Web API.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any, List, Union + +from ...core.errors import MetadataError, ValidationError +from ...core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND +from ...models.batch import BatchResult +from ...data._raw_request import _RawRequest +from ...data._batch_base import ( + _BatchBase, + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, + _ChangeSetBatchItem, + _MAX_BATCH_SIZE, +) + +if TYPE_CHECKING: + from ._async_odata import _AsyncODataClient + +__all__ = [] + + +# --------------------------------------------------------------------------- +# Batch client: resolves intents → raw requests → multipart body → HTTP → result +# --------------------------------------------------------------------------- + + +class _AsyncBatchClient(_BatchBase): + """ + Async version of the Dataverse batch client. + + Serialises a list of intent objects into an OData ``$batch`` multipart/mixed + request, dispatches it asynchronously, and parses the response. + + :param od: The active async OData client (provides helpers and HTTP transport). + """ + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + async def execute( + self, + items: List[Any], + continue_on_error: bool = False, + ) -> BatchResult: + """ + Resolve all intent objects, build the batch body, send it, and return results. + + Metadata pre-resolution (GET calls for MetadataId) happens here, asynchronously, + before the multipart body is assembled. + """ + if not items: + return BatchResult() + + resolved = await self._resolve_all(items) + + total = sum(len(r.requests) if isinstance(r, _ChangeSetBatchItem) else 1 for r in resolved) + if total > _MAX_BATCH_SIZE: + raise ValidationError( + f"Batch contains {total} operations, which exceeds the limit of " + f"{_MAX_BATCH_SIZE}. Split into multiple batches.", + subcode="batch_size_exceeded", + details={"count": total, "max": _MAX_BATCH_SIZE}, + ) + + batch_boundary = f"batch_{uuid.uuid4()}" + body = self._build_batch_body(resolved, batch_boundary) + + headers: dict[str, str] = { + "Content-Type": f'multipart/mixed; boundary="{batch_boundary}"', + } + if continue_on_error: + headers["Prefer"] = "odata.continue-on-error" + + url = f"{self._od.api}/$batch" + r = await self._od._request( + "post", + url, + data=body.encode("utf-8"), + headers=headers, + # 400 is expected: Dataverse returns 400 for top-level batch + # errors (e.g. malformed body). We parse the response body to + # surface the service error via _parse_batch_response / + # _raise_top_level_batch_error rather than letting _request raise. + expected=(200, 202, 207, 400), + ) + + return self._parse_batch_response(r) + + # ------------------------------------------------------------------ + # Intent resolution dispatcher + # ------------------------------------------------------------------ + + async def _resolve_all(self, items: List[Any]) -> List[Union[_RawRequest, _ChangeSetBatchItem]]: + result: List[Union[_RawRequest, _ChangeSetBatchItem]] = [] + for item in items: + if isinstance(item, _ChangeSet): + if not item.operations: + # Empty changeset — nothing to send; skip silently. + continue + cs_requests: List[_RawRequest] = [] + for op in item.operations: + cs_requests.append(await self._resolve_one(op)) + result.append(_ChangeSetBatchItem(requests=cs_requests)) + else: + result.extend(await self._resolve_item(item)) + return result + + async def _resolve_item(self, item: Any) -> List[_RawRequest]: + """Resolve a single intent to one or more _RawRequest objects.""" + if isinstance(item, _RecordCreate): + return await self._resolve_record_create(item) + if isinstance(item, _RecordUpdate): + return await self._resolve_record_update(item) + if isinstance(item, _RecordDelete): + return await self._resolve_record_delete(item) + if isinstance(item, _RecordGet): + return await self._resolve_record_get(item) + if isinstance(item, _RecordList): + return await self._resolve_record_list(item) + if isinstance(item, _RecordUpsert): + return await self._resolve_record_upsert(item) + if isinstance(item, _TableCreate): + return self._resolve_table_create(item) # sync; inherited from _BatchBase + if isinstance(item, _TableDelete): + return await self._resolve_table_delete(item) + if isinstance(item, _TableGet): + return self._resolve_table_get(item) # sync; inherited from _BatchBase + if isinstance(item, _TableList): + return self._resolve_table_list(item) # sync; inherited from _BatchBase + if isinstance(item, _TableAddColumns): + return await self._resolve_table_add_columns(item) + if isinstance(item, _TableRemoveColumns): + return await self._resolve_table_remove_columns(item) + if isinstance(item, _TableCreateOneToMany): + return self._resolve_table_create_one_to_many(item) # sync; inherited from _BatchBase + if isinstance(item, _TableCreateManyToMany): + return self._resolve_table_create_many_to_many(item) # sync; inherited from _BatchBase + if isinstance(item, _TableDeleteRelationship): + return self._resolve_table_delete_relationship(item) # sync; inherited from _BatchBase + if isinstance(item, _TableGetRelationship): + return self._resolve_table_get_relationship(item) # sync; inherited from _BatchBase + if isinstance(item, _TableCreateLookupField): + return self._resolve_table_create_lookup_field(item) # sync; inherited from _BatchBase + if isinstance(item, _QuerySql): + return await self._resolve_query_sql(item) + raise ValidationError( + f"Unknown batch item type: {type(item).__name__}", + subcode="unknown_batch_item", + ) + + async def _resolve_one(self, item: Any) -> _RawRequest: + """Resolve a changeset operation to exactly one _RawRequest.""" + resolved = await self._resolve_item(item) + if len(resolved) != 1: + raise ValidationError( + "Changeset operations must each produce exactly one HTTP request.", + subcode="changeset_multi_request", + ) + return resolved[0] + + # ------------------------------------------------------------------ + # Record resolvers — delegate to _AsyncODataClient._build_* methods + # ------------------------------------------------------------------ + + async def _resolve_record_create(self, op: _RecordCreate) -> List[_RawRequest]: + entity_set = await self._od._entity_set_from_schema_name(op.table) + if isinstance(op.data, dict): + return [await self._od._build_create(entity_set, op.table, op.data, content_id=op.content_id)] + return [await self._od._build_create_multiple(entity_set, op.table, op.data)] + + async def _resolve_record_update(self, op: _RecordUpdate) -> List[_RawRequest]: + if isinstance(op.ids, str): + if not isinstance(op.changes, dict): + raise TypeError("For single id, changes must be a dict") + return [await self._od._build_update(op.table, op.ids, op.changes, content_id=op.content_id)] + entity_set = await self._od._entity_set_from_schema_name(op.table) + return [await self._od._build_update_multiple(entity_set, op.table, op.ids, op.changes)] + + async def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: + if isinstance(op.ids, str): + return [await self._od._build_delete(op.table, op.ids, content_id=op.content_id)] + ids = [rid for rid in op.ids if rid] + if not ids: + return [] + if op.use_bulk_delete: + return [await self._od._build_delete_multiple(op.table, ids)] + return [await self._od._build_delete(op.table, rid) for rid in ids] + + async def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: + return [ + await self._od._build_get( + op.table, + op.record_id, + select=op.select, + expand=op.expand, + include_annotations=op.include_annotations, + ) + ] + + async def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: + return [ + await self._od._build_list( + op.table, + select=op.select, + filter=op.filter, + orderby=op.orderby, + top=op.top, + expand=op.expand, + page_size=op.page_size, + count=op.count, + include_annotations=op.include_annotations, + ) + ] + + async def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: + entity_set = await self._od._entity_set_from_schema_name(op.table) + if len(op.items) == 1: + item = op.items[0] + return [await self._od._build_upsert(entity_set, op.table, item.alternate_key, item.record)] + alternate_keys = [i.alternate_key for i in op.items] + records = [i.record for i in op.items] + return [await self._od._build_upsert_multiple(entity_set, op.table, alternate_keys, records)] + + # ------------------------------------------------------------------ + # Table resolvers — delegate to _AsyncODataClient._build_* methods + # (pre-resolution GETs for MetadataId remain here; they are batch- + # specific lookups needed before the relevant _build_* call) + # ------------------------------------------------------------------ + + async def _require_entity_metadata(self, table: str) -> str: + """Look up MetadataId for *table*, raising MetadataError if not found.""" + ent = await self._od._get_entity_by_table_schema_name(table) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + return ent["MetadataId"] + + async def _resolve_table_delete(self, op: _TableDelete) -> List[_RawRequest]: + metadata_id = await self._require_entity_metadata(op.table) + return [self._od._build_delete_entity(metadata_id)] + + async def _resolve_table_add_columns(self, op: _TableAddColumns) -> List[_RawRequest]: + metadata_id = await self._require_entity_metadata(op.table) + return [self._od._build_create_column(metadata_id, col_name, dtype) for col_name, dtype in op.columns.items()] + + async def _resolve_table_remove_columns(self, op: _TableRemoveColumns) -> List[_RawRequest]: + columns = [op.columns] if isinstance(op.columns, str) else list(op.columns) + metadata_id = await self._require_entity_metadata(op.table) + attr_metas = [ + await self._od._get_attribute_metadata(metadata_id, col_name, extra_select="@odata.type,AttributeType") + for col_name in columns + ] + requests: List[_RawRequest] = [] + for col_name, attr_meta in zip(columns, attr_metas): + if not attr_meta or not attr_meta.get("MetadataId"): + raise MetadataError( + f"Column '{col_name}' not found on table '{op.table}'.", + subcode=METADATA_COLUMN_NOT_FOUND, + ) + requests.append(self._od._build_delete_column(metadata_id, attr_meta["MetadataId"])) + return requests + + # ------------------------------------------------------------------ + # Query resolvers — delegate to _AsyncODataClient._build_* methods + # ------------------------------------------------------------------ + + async def _resolve_query_sql(self, op: _QuerySql) -> List[_RawRequest]: + return [await self._od._build_sql(op.sql)] diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_odata.py b/src/PowerPlatform/Dataverse/aio/data/_async_odata.py new file mode 100644 index 00000000..08bd9327 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_odata.py @@ -0,0 +1,1835 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async Dataverse Web API client with CRUD, SQL query, and table/column metadata management.""" + +from __future__ import annotations + +__all__ = [] + +import asyncio +import json +import time +import warnings +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Union + +if TYPE_CHECKING: + import aiohttp + +from urllib.parse import quote as _url_quote + +from ..core._async_http import _AsyncHttpClient, _AsyncResponse +from ._async_upload import _AsyncFileUploadMixin +from ._async_relationships import _AsyncRelationshipOperationsMixin +from ...core.errors import * +from ...data._raw_request import _RawRequest +from ...core._error_codes import ( + _http_subcode, + _is_transient_status, + VALIDATION_SQL_NOT_STRING, + VALIDATION_SQL_EMPTY, + VALIDATION_UNSUPPORTED_COLUMN_TYPE, + METADATA_ENTITYSET_NOT_FOUND, + METADATA_ENTITYSET_NAME_MISSING, + METADATA_TABLE_NOT_FOUND, + METADATA_TABLE_ALREADY_EXISTS, + METADATA_COLUMN_NOT_FOUND, +) + +from ...data._odata_base import ( + _ODataBase, + _GUID_RE, + _extract_pagingcookie, + _USER_AGENT, + _DEFAULT_EXPECTED_STATUSES, + _RequestContext, +) + + +class _AsyncODataClient(_AsyncFileUploadMixin, _AsyncRelationshipOperationsMixin, _ODataBase): + """Async Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" + + def __init__( + self, + auth, + base_url: str, + config=None, + session: Optional[aiohttp.ClientSession] = None, + ) -> None: + """Initialize the async OData client. + + Sets up authentication, base URL, configuration, and internal caches. + + :param auth: Async authentication manager providing ``_acquire_token(scope)`` that returns an object with ``access_token``. + :type auth: ~PowerPlatform.Dataverse.aio.core._async_auth._AsyncAuthManager + :param base_url: Organization base URL (e.g. ``"https://.crm.dynamics.com"``). + :type base_url: ``str`` + :param config: Optional Dataverse configuration (HTTP retry, backoff, timeout, language code). If omitted ``DataverseConfig.from_env()`` is used. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig | ``None`` + :param session: ``aiohttp.ClientSession`` for HTTP connection pooling. Must remain open for the lifetime of this client. + :type session: :class:`aiohttp.ClientSession` | ``None`` + :raises ValueError: If ``base_url`` is empty after stripping. + """ + super().__init__(base_url, config) + self.auth = auth + self._http = _AsyncHttpClient( + retries=self.config.http_retries, + backoff=self.config.http_backoff, + timeout=self.config.http_timeout, + session=session, + logger=self._http_logger, + ) + # Prevents concurrent coroutines from racing through the picklist TTL check + # and issuing redundant metadata fetches. + self._picklist_cache_lock: asyncio.Lock = asyncio.Lock() + + async def close(self) -> None: + """Close the async OData client and release resources. + + Clears all internal caches and closes the underlying HTTP client. + Safe to call multiple times. + """ + super().close() # sync: clears caches, closes logger + if self._http is not None: + await self._http.close() + + async def _headers(self) -> Dict[str, str]: + """Build standard OData headers with bearer auth.""" + scope = f"{self.base_url}/.default" + token = (await self.auth._acquire_token(scope)).access_token + ua = f"{_USER_AGENT} ({self._operation_context})" if self._operation_context else _USER_AGENT + return { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + "OData-MaxVersion": "4.0", + "OData-Version": "4.0", + "User-Agent": ua, + } + + async def _raw_request(self, method: str, url: str, **kwargs) -> _AsyncResponse: + return await self._http._request(method, url, **kwargs) + + async def _request( + self, + method: str, + url: str, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + **kwargs, + ) -> _AsyncResponse: + # Acquire base headers once (async), then use a sync closure for _RequestContext.build. + # _RequestContext.build is a sync classmethod defined in _ODataBase and shared by both + # sync and async clients — keeping it sync avoids duplicating the header-injection logic. + base_headers = await self._headers() + + def _merge(h: Optional[Dict[str, str]]) -> Dict[str, str]: + if not h: + return base_headers.copy() + merged = base_headers.copy() + merged.update(h) + return merged + + request_context = _RequestContext.build( + method, + url, + expected=expected, + merge_headers=_merge, + **kwargs, + ) + + r = await self._raw_request(request_context.method, request_context.url, **request_context.kwargs) + if r.status in request_context.expected: + return r + + response_headers = getattr(r, "headers", {}) or {} + raw_text = r.text + body_excerpt = raw_text[:200] + svc_code = None + msg = f"HTTP {r.status}" + try: + data = json.loads(raw_text) if raw_text else {} + if isinstance(data, dict): + inner = data.get("error") + if isinstance(inner, dict): + svc_code = inner.get("code") + imsg = inner.get("message") + if isinstance(imsg, str) and imsg.strip(): + msg = imsg.strip() + else: + imsg2 = data.get("message") + if isinstance(imsg2, str) and imsg2.strip(): + msg = imsg2.strip() + except Exception: + pass + sc = r.status + subcode = _http_subcode(sc) + request_id = ( + response_headers.get("x-ms-service-request-id") + or response_headers.get("req_id") + or response_headers.get("x-ms-request-id") + ) + traceparent = response_headers.get("traceparent") + ra = response_headers.get("Retry-After") + retry_after = None + if ra: + try: + retry_after = int(ra) + except Exception: + retry_after = None + is_transient = _is_transient_status(sc) + raise HttpError( + msg, + status_code=sc, + subcode=subcode, + service_error_code=svc_code, + correlation_id=request_context.headers.get( + "x-ms-correlation-id" + ), # this is a value set on client side, although it's logged on server side too + client_request_id=request_context.headers.get( + "x-ms-client-request-id" + ), # this is a value set on client side, although it's logged on server side too + service_request_id=request_id, + traceparent=traceparent, + body_excerpt=body_excerpt, + retry_after=retry_after, + is_transient=is_transient, + ) + + async def _execute_raw( + self, + req: _RawRequest, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + ) -> _AsyncResponse: + """Execute a ``_RawRequest`` and return the HTTP response. + + Encodes the pre-serialised body (if present) as UTF-8 and merges any + per-request headers into the standard OData header set. + """ + kwargs: Dict[str, Any] = {} + if req.body is not None: + kwargs["data"] = req.body.encode("utf-8") + if req.headers: + kwargs["headers"] = req.headers + return await self._request(req.method.lower(), req.url, expected=expected, **kwargs) + + # --- CRUD Internal functions --- + async def _create(self, entity_set: str, table_schema_name: str, record: Dict[str, Any]) -> str: + """Create a single record and return its GUID. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param record: Attribute payload mapped by logical column names. + :type record: ``dict[str, Any]`` + + :return: Created record GUID. + :rtype: ``str`` + + .. note:: + Relies on ``OData-EntityId`` (canonical) or ``Location`` response header. No response body parsing is performed. Raises ``RuntimeError`` if neither header contains a GUID. + """ + r = await self._execute_raw(await self._build_create(entity_set, table_schema_name, record)) + ent_loc = r.headers.get("OData-EntityId") or r.headers.get("OData-EntityID") + if ent_loc: + m = _GUID_RE.search(ent_loc) + if m: + return m.group(0) + loc = r.headers.get("Location") + if loc: + m = _GUID_RE.search(loc) + if m: + return m.group(0) + header_keys = ", ".join(sorted(r.headers.keys())) + raise RuntimeError( + f"Create response missing GUID in OData-EntityId/Location headers (status={r.status}). Headers: {header_keys}" + ) + + async def _create_multiple( + self, + entity_set: str, + table_schema_name: str, + records: List[Dict[str, Any]], + ) -> List[str]: + """Create multiple records using the collection-bound ``CreateMultiple`` action. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param records: Payload dictionaries mapped by column schema names. + :type records: ``list[dict[str, Any]]`` + + :return: List of created record GUIDs (may be empty if response lacks IDs). + :rtype: ``list[str]`` + + .. note:: + Logical type stamping: if any payload omits ``@odata.type`` the client injects ``Microsoft.Dynamics.CRM.``. If all payloads already include ``@odata.type`` no modification occurs. + """ + if not all(isinstance(r, dict) for r in records): + raise TypeError("All items for multi-create must be dicts") + r = await self._execute_raw(await self._build_create_multiple(entity_set, table_schema_name, records)) + try: + body = r.json() + except ValueError: + body = {} + if not isinstance(body, dict): + return [] + # Expected: { "Ids": [guid, ...] } + ids = body.get("Ids") + if isinstance(ids, list): + return [i for i in ids if isinstance(i, str)] + + value = body.get("value") + if isinstance(value, list): + # Extract IDs if possible + out: List[str] = [] + for item in value: + if isinstance(item, dict): + # Heuristic: look for a property ending with 'id' + for k, v in item.items(): + if isinstance(k, str) and k.lower().endswith("id") and isinstance(v, str) and len(v) >= 32: + out.append(v) + break + return out + return [] + + async def _upsert( + self, + entity_set: str, + table_schema_name: str, + alternate_key: Dict[str, Any], + record: Dict[str, Any], + ) -> None: + """Upsert a single record using an alternate key. + + Issues a PATCH request to ``{entity_set}({key_pairs})`` where ``key_pairs`` + is the OData alternate key segment built from ``alternate_key``. Creates the + record if it does not exist; updates it if it does. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param alternate_key: Mapping of alternate key attribute names to their values + used to identify the target record in the URL. + :type alternate_key: ``dict[str, Any]`` + :param record: Attribute payload to set on the record. + :type record: ``dict[str, Any]`` + + :return: ``None`` + :rtype: ``None`` + """ + record = self._lowercase_keys(record) + record = await self._convert_labels_to_ints(table_schema_name, record) + key_str = self._build_alternate_key_str(alternate_key) + url = f"{self.api}/{entity_set}({key_str})" + await self._request("patch", url, json=record, expected=(200, 201, 204)) + + async def _upsert_multiple( + self, + entity_set: str, + table_schema_name: str, + alternate_keys: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ) -> None: + """Upsert multiple records using the collection-bound ``UpsertMultiple`` action. + + Each target is formed by merging the corresponding alternate key fields and record + fields. The ``@odata.type`` annotation is injected automatically if absent. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param alternate_keys: List of alternate key dictionaries, one per record. + Order is significant: ``alternate_keys[i]`` must correspond to ``records[i]``. + Python ``list`` preserves insertion order, so the correspondence is guaranteed + as long as both lists are built from the same source in the same order. + :type alternate_keys: ``list[dict[str, Any]]`` + :param records: List of record payload dictionaries, one per record. + Must be the same length as ``alternate_keys``. + :type records: ``list[dict[str, Any]]`` + + :return: ``None`` + :rtype: ``None`` + + :raises ValueError: If ``alternate_keys`` and ``records`` differ in length, or if + any record payload contains an alternate key field with a conflicting value. + """ + if len(alternate_keys) != len(records): + raise ValueError( + f"alternate_keys and records must have the same length " f"({len(alternate_keys)} != {len(records)})" + ) + logical_name = table_schema_name.lower() + lowered_records = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table_schema_name, r) for r in lowered_records] + targets: List[Dict[str, Any]] = [] + for alt_key, record_processed in zip(alternate_keys, converted): + alt_key_lower = self._lowercase_keys(alt_key) + conflicting = { + k for k in set(alt_key_lower) & set(record_processed) if alt_key_lower[k] != record_processed[k] + } + if conflicting: + raise ValueError(f"record payload conflicts with alternate_key on fields: {sorted(conflicting)!r}") + if "@odata.type" not in record_processed: + record_processed["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" + key_str = self._build_alternate_key_str(alt_key) + record_processed["@odata.id"] = f"{entity_set}({key_str})" + targets.append(record_processed) + payload = {"Targets": targets} + url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpsertMultiple" + await self._request("post", url, json=payload, expected=(200, 201, 204)) + + # --- Derived helpers for high-level client ergonomics --- + async def _primary_id_attr(self, table_schema_name: str) -> str: + """Return primary key attribute using metadata; error if unavailable.""" + cache_key = self._normalize_cache_key(table_schema_name) + pid = self._logical_primaryid_cache.get(cache_key) + if pid: + return pid + # Resolve metadata (populates _logical_primaryid_cache or raises if table_schema_name unknown) + await self._entity_set_from_schema_name(table_schema_name) + pid2 = self._logical_primaryid_cache.get(cache_key) + if pid2: + return pid2 + raise RuntimeError( + f"PrimaryIdAttribute not resolved for table_schema_name '{table_schema_name}'. Metadata did not include PrimaryIdAttribute." + ) + + async def _update_by_ids( + self, + table_schema_name: str, + ids: List[str], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """Update many records by GUID list using the collection-bound ``UpdateMultiple`` action. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param ids: GUIDs of target records. + :type ids: ``list[str]`` + :param changes: Broadcast patch (``dict``) applied to all IDs, or list of per-record patches (1:1 with ``ids``). + :type changes: ``dict`` | ``list[dict]`` + + :return: ``None`` + :rtype: ``None`` + """ + if not isinstance(ids, list): + raise TypeError("ids must be list[str]") + if not ids: + return None + pk_attr = await self._primary_id_attr(table_schema_name) + entity_set = await self._entity_set_from_schema_name(table_schema_name) + if isinstance(changes, dict): + batch = [{pk_attr: rid, **changes} for rid in ids] + await self._update_multiple(entity_set, table_schema_name, batch) + return + if not isinstance(changes, list): + raise TypeError("changes must be dict or list[dict]") + if len(changes) != len(ids): + raise ValueError("Length of changes list must match length of ids list") + batch: List[Dict[str, Any]] = [] + for rid, patch in zip(ids, changes): + if not isinstance(patch, dict): + raise TypeError("Each patch must be a dict") + batch.append({pk_attr: rid, **patch}) + await self._update_multiple(entity_set, table_schema_name, batch) + + async def _delete_multiple( + self, + table_schema_name: str, + ids: List[str], + ) -> Optional[str]: + """Delete many records by GUID list via the ``BulkDelete`` action. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param ids: GUIDs of records to delete. + :type ids: ``list[str]`` + + :return: BulkDelete asynchronous job identifier when executed in bulk; ``None`` if no IDs provided or single deletes performed. + :rtype: ``str`` | ``None`` + """ + targets = [rid for rid in ids if rid] + if not targets: + return None + response = await self._execute_raw( + await self._build_delete_multiple(table_schema_name, targets), + expected=(200, 202, 204), + ) + job_id = None + try: + body = response.json() + except ValueError: + body = {} + if isinstance(body, dict): + job_id = body.get("JobId") + return job_id + + async def _update(self, table_schema_name: str, key: str, data: Dict[str, Any]) -> None: + """Update an existing record by GUID. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses). + :type key: ``str`` + :param data: Partial entity payload (attributes to patch). + :type data: ``dict[str, Any]`` + :return: ``None`` + :rtype: ``None`` + """ + await self._execute_raw(await self._build_update(table_schema_name, key, data)) + + async def _update_multiple( + self, + entity_set: str, + table_schema_name: str, + records: List[Dict[str, Any]], + ) -> None: + """Bulk update existing records via the collection-bound ``UpdateMultiple`` action. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table, e.g. "new_MyTestTable". + :type table_schema_name: ``str`` + :param records: List of patch dictionaries. Each must include the true primary key attribute (e.g. ``accountid``) and one or more fields to update. + :type records: ``list[dict[str, Any]]`` + :return: ``None`` + :rtype: ``None`` + + .. note:: + - Endpoint: ``POST /{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple`` with body ``{"Targets": [...]}``. + - Transactional semantics: if any individual update fails, the entire request rolls back. + - Response content is ignored; no stable contract for returned IDs/representations. + - Caller must supply the correct primary key attribute (e.g. ``accountid``) in every record. + """ + if not isinstance(records, list) or not records or not all(isinstance(r, dict) for r in records): + raise TypeError("records must be a non-empty list[dict]") + await self._execute_raw(await self._build_update_multiple_from_records(entity_set, table_schema_name, records)) + + async def _delete(self, table_schema_name: str, key: str) -> None: + """Delete a record by GUID. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses) + :type key: ``str`` + + :return: ``None`` + :rtype: ``None`` + """ + await self._execute_raw(await self._build_delete(table_schema_name, key)) + + async def _get( + self, + table_schema_name: str, + key: str, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Dict[str, Any]: + """Retrieve a single record. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses). + :type key: ``str`` + :param select: Columns to select; joined with commas into $select. + :type select: ``list[str]`` | ``None`` + :param expand: Navigation properties to expand (``$expand``); passed as-is (case-sensitive). + :type expand: ``list[str]`` | ``None`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: ``str`` | ``None`` + + :return: Retrieved record dictionary (may be empty if no selected attributes). + :rtype: ``dict[str, Any]`` + """ + r = await self._execute_raw( + await self._build_get( + table_schema_name, key, select=select, expand=expand, include_annotations=include_annotations + ) + ) + return r.json() + + async def _get_multiple( + self, + table_schema_name: str, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> AsyncIterator[List[Dict[str, Any]]]: + """Iterate records from an entity set, yielding one page (list of dicts) at a time. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param select: Columns to include (``$select``) or ``None``. Column names are automatically lowercased. + :type select: ``list[str]`` | ``None`` + :param filter: OData ``$filter`` expression or ``None``. This is passed as-is without transformation. Users must provide lowercase logical column names (e.g., "statecode eq 0"). + :type filter: ``str`` | ``None`` + :param orderby: Order expressions (``$orderby``) or ``None``. Column names are automatically lowercased. + :type orderby: ``list[str]`` | ``None`` + :param top: Max total records (applied on first request as ``$top``) or ``None``. + :type top: ``int`` | ``None`` + :param expand: Navigation properties to expand (``$expand``) or ``None``. These are case-sensitive and passed as-is. Users must provide exact navigation property names from entity metadata. + :type expand: ``list[str]`` | ``None`` + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: ``int`` | ``None`` + :param count: If ``True``, adds ``$count=true`` to include a total record count in the response. + :type count: ``bool`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: ``str`` | ``None`` + + :return: Async iterator yielding pages (each page is a ``list`` of record dicts). + :rtype: ``AsyncIterator[list[dict[str, Any]]]`` + """ + extra_headers: Dict[str, str] = {} + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + if prefer_parts: + extra_headers["Prefer"] = ",".join(prefer_parts) + + async def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + headers = extra_headers if extra_headers else None + r = await self._request("get", url, headers=headers, params=params) + try: + return r.json() + except ValueError: + return {} + + entity_set = await self._entity_set_from_schema_name(table_schema_name) + base_url = f"{self.api}/{entity_set}" + params: Dict[str, Any] = {} + if select: + # Lowercase column names for case-insensitive matching + params["$select"] = ",".join(self._lowercase_list(select)) + if filter: + # Filter is passed as-is; users must use lowercase column names in filter expressions + params["$filter"] = filter + if orderby: + # Lowercase column names for case-insensitive matching + params["$orderby"] = ",".join(self._lowercase_list(orderby)) + if expand: + # Lowercase navigation property names for case-insensitive matching + params["$expand"] = ",".join(expand) + if top is not None: + params["$top"] = int(top) + if count: + params["$count"] = "true" + + data = await _do_request(base_url, params=params) + items = data.get("value") if isinstance(data, dict) else None + if isinstance(items, list) and items: + yield [x for x in items if isinstance(x, dict)] + + next_link = None + if isinstance(data, dict): + next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") + + while next_link: + data = await _do_request(next_link) + items = data.get("value") if isinstance(data, dict) else None + if isinstance(items, list) and items: + yield [x for x in items if isinstance(x, dict)] + next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") if isinstance(data, dict) else None + + # --------------------------- SQL Custom API ------------------------- + async def _query_sql(self, sql: str) -> list[dict[str, Any]]: + """Execute a read-only SQL SELECT using the Dataverse Web API ``?sql=`` capability. + + :param sql: Single SELECT statement within the supported subset. + :type sql: ``str`` + + :return: Result rows (empty list if none). + :rtype: ``list[dict[str, Any]]`` + + :raises ValidationError: If ``sql`` is not a ``str`` or is empty. + :raises MetadataError: If logical table name resolution fails. + + .. note:: + Endpoint form: ``GET /{entity_set}?sql=``. The client + extracts the logical table name, resolves the entity set (metadata + cached), then issues the request. ``SELECT *`` raises + :class:`~PowerPlatform.Dataverse.core.errors.ValidationError` -- + it is deliberately rejected, not silently rewritten. + """ + if not isinstance(sql, str): + raise ValidationError("sql must be a string", subcode=VALIDATION_SQL_NOT_STRING) + if not sql.strip(): + raise ValidationError("sql must be a non-empty string", subcode=VALIDATION_SQL_EMPTY) + sql = sql.strip() + + # Apply safety guardrails (block unsupported syntax including writes, + # warn on risky patterns). SELECT * raises ValidationError here before + # any table resolution. + sql = self._sql_guardrails(sql) + + r = await self._execute_raw(await self._build_sql(sql)) + try: + body = r.json() + except ValueError: + return [] + + # Collect first page + results: list[dict[str, Any]] = [] + if isinstance(body, list): + return [row for row in body if isinstance(row, dict)] + if not isinstance(body, dict): + return results + + value = body.get("value") + if isinstance(value, list): + results = [row for row in value if isinstance(row, dict)] + + # Follow pagination links until exhausted + raw_link = body.get("@odata.nextLink") or body.get("odata.nextLink") + next_link: str | None = raw_link if isinstance(raw_link, str) else None + visited: set[str] = set() + seen_cookies: set[str] = set() + while next_link: + # Guard 1: exact URL cycle (same next_link returned twice) + if next_link in visited: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + "the Dataverse server returned the same nextLink URL twice, " + "indicating an infinite pagination cycle. " + "Returning the rows collected so far. " + "To avoid pagination entirely, add a TOP clause to your query.", + RuntimeWarning, + stacklevel=4, + ) + break + visited.add(next_link) + # Guard 2: server-side bug where pagingcookie does not advance between + # pages (pagenumber increments but cookie GUIDs stay the same), which + # causes an infinite loop even though URLs differ. + cookie = _extract_pagingcookie(next_link) + if cookie is not None: + if cookie in seen_cookies: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + "the Dataverse server returned the same pagingcookie twice " + "(pagenumber incremented but the paging position did not advance). " + "This is a server-side bug. Returning the rows collected so far. " + "To avoid pagination entirely, add a TOP clause to your query.", + RuntimeWarning, + stacklevel=4, + ) + break + seen_cookies.add(cookie) + try: + page_resp = await self._request("get", next_link) + except Exception as exc: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + f"the next-page request failed: {exc}. " + "Add a TOP clause to your query to limit results to a single page.", + RuntimeWarning, + stacklevel=5, + ) + break + try: + page_body = page_resp.json() + except ValueError as exc: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + f"the next-page response was not valid JSON: {exc}. " + "Add a TOP clause to your query to limit results to a single page.", + RuntimeWarning, + stacklevel=5, + ) + break + if not isinstance(page_body, dict): + break + page_value = page_body.get("value") + if not isinstance(page_value, list) or not page_value: + break + results.extend(row for row in page_value if isinstance(row, dict)) + raw_link = page_body.get("@odata.nextLink") or page_body.get("odata.nextLink") + next_link = raw_link if isinstance(raw_link, str) else None + + return results + + # ---------------------- Entity set resolution ----------------------- + async def _entity_set_from_schema_name(self, table_schema_name: str) -> str: + """Resolve entity set name (plural) from a schema name (singular) name using metadata. + + Caches results for subsequent queries. Case-insensitive. + """ + if not table_schema_name: + raise ValueError("table schema name required") + + # Use normalized (lowercase) key for cache lookup + cache_key = self._normalize_cache_key(table_schema_name) + cached = self._logical_to_entityset_cache.get(cache_key) + if cached: + return cached + url = f"{self.api}/EntityDefinitions" + # LogicalName in Dataverse is stored in lowercase, so we need to lowercase for the filter + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + params = { + "$select": "LogicalName,EntitySetName,PrimaryIdAttribute", + "$filter": f"LogicalName eq '{logical_escaped}'", + } + r = await self._request("get", url, params=params) + try: + body = r.json() + items = body.get("value", []) if isinstance(body, dict) else [] + except ValueError: + items = [] + if not items: + plural_hint = ( + " (did you pass a plural entity set name instead of the singular table schema name?)" + if table_schema_name.endswith("s") and not table_schema_name.endswith("ss") + else "" + ) + raise MetadataError( + f"Unable to resolve entity set for table schema name '{table_schema_name}'. Provide the singular table schema name.{plural_hint}", + subcode=METADATA_ENTITYSET_NOT_FOUND, + ) + md = items[0] + es = md.get("EntitySetName") + if not es: + raise MetadataError( + f"Metadata response missing EntitySetName for table schema name '{table_schema_name}'.", + subcode=METADATA_ENTITYSET_NAME_MISSING, + ) + self._logical_to_entityset_cache[cache_key] = es + primary_id_attr = md.get("PrimaryIdAttribute") + if isinstance(primary_id_attr, str) and primary_id_attr: + self._logical_primaryid_cache[cache_key] = primary_id_attr + return es + + # ---------------------- Table metadata helpers ---------------------- + async def _get_entity_by_table_schema_name( + self, + table_schema_name: str, + headers: Optional[Dict[str, str]] = None, + ) -> Optional[Dict[str, Any]]: + """Get entity metadata by table schema name. Case-insensitive. + + Note: LogicalName is stored lowercase in Dataverse, so we lowercase the input + for case-insensitive matching. The response includes SchemaName, LogicalName, + EntitySetName, and MetadataId. + """ + url = f"{self.api}/EntityDefinitions" + # LogicalName is stored lowercase, so we lowercase the input for lookup + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + params = { + "$select": "MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute", + "$filter": f"LogicalName eq '{logical_escaped}'", + } + r = await self._request("get", url, params=params, headers=headers) + items = (r.json()).get("value", []) + return items[0] if items else None + + async def _create_entity( + self, + table_schema_name: str, + display_name: str, + attributes: List[Dict[str, Any]], + solution_unique_name: Optional[str] = None, + ) -> Dict[str, Any]: + url = f"{self.api}/EntityDefinitions" + payload = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": table_schema_name, + "DisplayName": self._label(display_name), + "DisplayCollectionName": self._label(display_name + "s"), + "Description": self._label(f"Custom entity for {display_name}"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsActivity": False, + "Attributes": attributes, + } + params = None + if solution_unique_name: + params = {"SolutionUniqueName": solution_unique_name} + await self._request("post", url, json=payload, params=params) + ent = await self._get_entity_by_table_schema_name( + table_schema_name, + headers={"Consistency": "Strong"}, + ) + if not ent or not ent.get("EntitySetName"): + raise RuntimeError( + f"Failed to create or retrieve entity '{table_schema_name}' (EntitySetName not available)." + ) + if not ent.get("MetadataId"): + raise RuntimeError(f"MetadataId missing after creating entity '{table_schema_name}'.") + return ent + + async def _get_attribute_metadata( + self, + entity_metadata_id: str, + column_name: str, + extra_select: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + # Convert to lowercase logical name for lookup + logical_name = column_name.lower() + attr_escaped = self._escape_odata_quotes(logical_name) + url = f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes" + select_fields = ["MetadataId", "LogicalName", "SchemaName"] + if extra_select: + for piece in extra_select.split(","): + piece = piece.strip() + if not piece or piece in select_fields: + continue + if piece.startswith("@"): + continue + if piece not in select_fields: + select_fields.append(piece) + params = { + "$select": ",".join(select_fields), + "$filter": f"LogicalName eq '{attr_escaped}'", + } + r = await self._request("get", url, params=params) + try: + body = r.json() + except ValueError: + return None + items = body.get("value") if isinstance(body, dict) else None + if isinstance(items, list) and items: + item = items[0] + if isinstance(item, dict): + return item + return None + + async def _list_columns( + self, + table_schema_name: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """List all attribute (column) definitions for a table. + + Issues ``GET EntityDefinitions({MetadataId})/Attributes`` with optional + ``$select`` and ``$filter`` query parameters. + + :param table_schema_name: Schema name of the table + (e.g. ``"account"`` or ``"new_Product"``). + :type table_schema_name: ``str`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + :param filter: Optional OData ``$filter`` expression. For example, + ``"AttributeType eq 'String'"`` returns only string columns. + :type filter: ``str`` or ``None`` + + :return: List of raw attribute metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table is not found. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + metadata_id = ent["MetadataId"] + url = f"{self.api}/EntityDefinitions({metadata_id})/Attributes" + params: Dict[str, str] = {} + if select: + params["$select"] = ",".join(select) + if filter: + params["$filter"] = filter + r = await self._request("get", url, params=params) + return (r.json()).get("value", []) + + async def _wait_for_attribute_visibility( + self, + entity_set: str, + attribute_name: str, + delays: tuple = (0, 3, 10, 20), + ) -> None: + """Wait for a newly created attribute to become visible in the data API. + + After creating an attribute via the metadata API, there can be a delay before + it becomes queryable in the data API. This method polls the entity set with + the attribute in the $select clause until it succeeds or all delays are exhausted. + """ + # Convert to lowercase logical name for URL + logical_name = attribute_name.lower() + probe_url = f"{self.api}/{entity_set}?$top=1&$select={logical_name}" + last_error = None + total_wait = sum(delays) + + for delay in delays: + if delay: + await asyncio.sleep(delay) + try: + await self._request("get", probe_url) + return + except Exception as ex: + last_error = ex + continue + + # All retries exhausted - raise with context + raise RuntimeError( + f"Attribute '{logical_name}' did not become visible in the data API " + f"after {total_wait} seconds (exhausted all retries)." + ) from last_error + + async def _request_metadata_with_retry(self, method: str, url: str, **kwargs) -> _AsyncResponse: + """Fetch metadata with retries on transient errors.""" + max_attempts = 5 + backoff_seconds = 0.4 + for attempt in range(1, max_attempts + 1): + try: + return await self._request(method, url, **kwargs) + except HttpError as err: + if getattr(err, "status_code", None) == 404: + if attempt < max_attempts: + await asyncio.sleep(backoff_seconds * (2 ** (attempt - 1))) + continue + raise RuntimeError(f"Metadata request failed after {max_attempts} retries (404): {url}") from err + raise + + async def _bulk_fetch_picklists(self, table_schema_name: str) -> None: + """Fetch all picklist attributes and their options for a table in one API call. + + Uses collection-level PicklistAttributeMetadata cast to retrieve every picklist + attribute on the table, including its OptionSet options. Populates the nested + cache so that ``_convert_labels_to_ints`` resolves labels without further API calls. + The Dataverse metadata API does not page results. + """ + table_key = self._normalize_cache_key(table_schema_name) + + # Fast path: skip the lock when the cache is already warm. + now = time.time() + table_entry = self._picklist_label_cache.get(table_key) + if isinstance(table_entry, dict) and (now - table_entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: + return + + # Slow path: acquire the lock so that only one coroutine issues the metadata + # fetch. The second TTL check inside the lock handles the case where another + # coroutine populated the cache while we were waiting. + async with self._picklist_cache_lock: + now = time.time() + table_entry = self._picklist_label_cache.get(table_key) + if isinstance(table_entry, dict) and (now - table_entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: + return + + table_esc = self._escape_odata_quotes(table_schema_name.lower()) + url = ( + f"{self.api}/EntityDefinitions(LogicalName='{table_esc}')" + f"/Attributes/Microsoft.Dynamics.CRM.PicklistAttributeMetadata" + f"?$select=LogicalName&$expand=OptionSet($select=Options)" + ) + response = await self._request_metadata_with_retry("get", url) + body = response.json() + items = body.get("value", []) if isinstance(body, dict) else [] + + picklists: Dict[str, Dict[str, int]] = {} + for item in items: + if not isinstance(item, dict): + continue + ln = item.get("LogicalName", "").lower() + if not ln: + continue + option_set = item.get("OptionSet") or {} + options = option_set.get("Options") if isinstance(option_set, dict) else None + mapping: Dict[str, int] = {} + if isinstance(options, list): + for opt in options: + if not isinstance(opt, dict): + continue + val = opt.get("Value") + if not isinstance(val, int): + continue + label_def = opt.get("Label") or {} + locs = label_def.get("LocalizedLabels") + if isinstance(locs, list): + for loc in locs: + if isinstance(loc, dict): + lab = loc.get("Label") + if isinstance(lab, str) and lab.strip(): + normalized = self._normalize_picklist_label(lab) + mapping.setdefault(normalized, val) + picklists[ln] = mapping + + self._picklist_label_cache[table_key] = {"ts": now, "picklists": picklists} + + async def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any]) -> Dict[str, Any]: + """Return a copy of record with any labels converted to option ints. + + Heuristic: For each string value, attempt to resolve against picklist metadata. + If attribute isn't a picklist or label not found, value left unchanged. + + On first encounter of a table, bulk-fetches all picklist attributes and + their options in a single API call, then resolves labels from the warm cache. + """ + resolved_record = record.copy() + + # Check if there are any string-valued candidates worth resolving + has_candidates = any( + isinstance(v, str) and v.strip() and isinstance(k, str) and "@odata." not in k + for k, v in resolved_record.items() + ) + if not has_candidates: + return resolved_record + + # Bulk-fetch all picklists for this table (1 API call, cached for TTL) + await self._bulk_fetch_picklists(table_schema_name) + + # Resolve labels from the nested cache + table_key = self._normalize_cache_key(table_schema_name) + table_entry = self._picklist_label_cache.get(table_key) + if not isinstance(table_entry, dict): + return resolved_record + picklists = table_entry.get("picklists", {}) + + for k, v in resolved_record.items(): + if not isinstance(v, str) or not v.strip(): + continue + if isinstance(k, str) and "@odata." in k: + continue + attr_key = self._normalize_cache_key(k) + mapping = picklists.get(attr_key) + if not isinstance(mapping, dict) or not mapping: + continue + norm = self._normalize_picklist_label(v) + val = mapping.get(norm) + if val is not None: + resolved_record[k] = val + return resolved_record + + async def _get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: + """Return basic metadata for a custom table if it exists. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: Metadata summary or ``None`` if not found. + :rtype: ``dict[str, Any]`` | ``None`` + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent: + return None + return { + "table_schema_name": ent.get("SchemaName") or table_schema_name, + "table_logical_name": ent.get("LogicalName"), + "entity_set_name": ent.get("EntitySetName"), + "metadata_id": ent.get("MetadataId"), + "primary_name_attribute": ent.get("PrimaryNameAttribute"), + "primary_id_attribute": ent.get("PrimaryIdAttribute"), + "columns_created": [], + } + + async def _list_tables( + self, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all non-private tables (``IsPrivate eq false``). + + :param filter: Optional additional OData ``$filter`` expression that is + combined with the default ``IsPrivate eq false`` clause using + ``and``. For example, ``"SchemaName eq 'Account'"`` becomes + ``"IsPrivate eq false and (SchemaName eq 'Account')"``. + When ``None`` (the default), only the ``IsPrivate eq false`` filter + is applied. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase) because + ``EntityDefinitions`` uses PascalCase property names. + When ``None`` (the default) or an empty list, no ``$select`` is + applied and all properties are returned. Passing a bare string + raises ``TypeError``. + :type select: ``list[str]`` or ``None`` + + :return: Metadata entries for non-private tables (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises HttpError: If the metadata request fails. + """ + r = await self._execute_raw(self._build_list_entities(filter=filter, select=select)) + return (r.json()).get("value", []) + + async def _delete_table(self, table_schema_name: str) -> None: + """Delete a table by schema name. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: ``None`` + :rtype: ``None`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the delete request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + await self._execute_raw(self._build_delete_entity(ent["MetadataId"])) + + # ------------------- Alternate key metadata helpers ------------------- + + async def _create_alternate_key( + self, + table_schema_name: str, + key_name: str, + columns: List[str], + display_name_label=None, + ) -> Dict[str, Any]: + """Create an alternate key on a table. + + Issues ``POST EntityDefinitions(LogicalName='{logical_name}')/Keys`` + with ``EntityKeyMetadata`` payload. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key_name: Schema name for the new alternate key. + :type key_name: ``str`` + :param columns: List of column logical names that compose the key. + :type columns: ``list[str]`` + :param display_name_label: Label for the key display name. + :type display_name_label: ``Label`` or ``None`` + + :return: Dictionary with ``metadata_id``, ``schema_name``, and ``key_attributes``. + :rtype: ``dict[str, Any]`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys" + payload: Dict[str, Any] = { + "SchemaName": key_name, + "KeyAttributes": columns, + } + if display_name_label is not None: + payload["DisplayName"] = display_name_label.to_dict() + r = await self._request("post", url, json=payload) + metadata_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "metadata_id": metadata_id, + "schema_name": key_name, + "key_attributes": columns, + } + + async def _get_alternate_keys(self, table_schema_name: str) -> List[Dict[str, Any]]: + """List all alternate keys on a table. + + Issues ``GET EntityDefinitions(LogicalName='{logical_name}')/Keys``. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: List of raw ``EntityKeyMetadata`` dictionaries. + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys" + r = await self._request("get", url) + return (r.json()).get("value", []) + + async def _delete_alternate_key(self, table_schema_name: str, key_id: str) -> None: + """Delete an alternate key by metadata ID. + + Issues ``DELETE EntityDefinitions(LogicalName='{logical_name}')/Keys({key_id})``. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key_id: Metadata GUID of the alternate key. + :type key_id: ``str`` + + :return: ``None`` + :rtype: ``None`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys({key_id})" + await self._request("delete", url) + + async def _create_table( + self, + table_schema_name: str, + schema: Dict[str, Any], + solution_unique_name: Optional[str] = None, + primary_column_schema_name: Optional[str] = None, + display_name: Optional[str] = None, + ) -> Dict[str, Any]: + """Create a custom table with specified columns. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param schema: Mapping of column name -> type spec (``str`` or ``Enum`` subclass). + :type schema: ``dict[str, Any]`` + :param solution_unique_name: Optional solution container for the new table; if provided must be non-empty. + :type solution_unique_name: ``str`` | ``None`` + :param primary_column_schema_name: Optional primary column schema name. + :type primary_column_schema_name: ``str`` | ``None`` + :param display_name: Human-readable display name for the table. Defaults to ``table_schema_name``. + :type display_name: ``str`` | ``None`` + + :return: Metadata summary for the created table including created column schema names. + :rtype: ``dict[str, Any]`` + + :raises MetadataError: If the table already exists. + :raises ValueError: If a column type is unsupported or ``solution_unique_name`` is empty. + :raises TypeError: If ``solution_unique_name`` is not a ``str`` when provided. + :raises HttpError: If underlying HTTP requests fail. + """ + # Check if table already exists (case-insensitive) + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if ent: + raise MetadataError( + f"Table '{table_schema_name}' already exists.", + subcode=METADATA_TABLE_ALREADY_EXISTS, + ) + + created_cols: List[str] = [] + + # Use provided primary column name, or derive from table_schema_name prefix (e.g., "new_Product" -> "new_Name"). + # If no prefix detected, default to "new_Name"; server will validate overall table schema. + if primary_column_schema_name: + primary_attr_schema = primary_column_schema_name + else: + primary_attr_schema = ( + f"{table_schema_name.split('_',1)[0]}_Name" if "_" in table_schema_name else "new_Name" + ) + + attributes: List[Dict[str, Any]] = [] + attributes.append(self._attribute_payload(primary_attr_schema, "string", is_primary_name=True)) + for col_name, dtype in schema.items(): + payload = self._attribute_payload(col_name, dtype) + if not payload: + raise ValueError(f"Unsupported column type '{dtype}' for '{col_name}'.") + attributes.append(payload) + created_cols.append(col_name) + + if solution_unique_name is not None: + if not isinstance(solution_unique_name, str): + raise TypeError("solution_unique_name must be a string when provided") + if not solution_unique_name: + raise ValueError("solution_unique_name cannot be empty") + + if display_name is not None: + if not isinstance(display_name, str) or not display_name.strip(): + raise TypeError("display_name must be a non-empty string when provided") + + metadata = await self._create_entity( + table_schema_name=table_schema_name, + display_name=display_name if display_name is not None else table_schema_name, + attributes=attributes, + solution_unique_name=solution_unique_name, + ) + + return { + "table_schema_name": table_schema_name, + "table_logical_name": metadata.get("LogicalName"), + "entity_set_name": metadata.get("EntitySetName"), + "metadata_id": metadata.get("MetadataId"), + "primary_name_attribute": metadata.get("PrimaryNameAttribute"), + "primary_id_attribute": metadata.get("PrimaryIdAttribute"), + "columns_created": created_cols, + } + + async def _create_columns( + self, + table_schema_name: str, + columns: Dict[str, Any], + ) -> List[str]: + """Create new columns on an existing table. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param columns: Mapping of column schema name -> type spec (``str`` or ``Enum`` subclass). + :type columns: ``dict[str, Any]`` + + :return: List of created column schema names. + :rtype: ``list[str]`` + + :raises TypeError: If ``columns`` is not a non-empty dict. + :raises MetadataError: If the target table does not exist. + :raises ValueError: If a column type is unsupported. + :raises HttpError: If an underlying HTTP request fails. + """ + if not isinstance(columns, dict) or not columns: + raise TypeError("columns must be a non-empty dict[name -> type]") + + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + metadata_id = ent.get("MetadataId") + created: List[str] = [] + needs_picklist_flush = False + + for column_name, column_type in columns.items(): + attr = self._attribute_payload(column_name, column_type) + if not attr: + raise ValidationError( + f"Unsupported column type '{column_type}' for column '{column_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + if "OptionSet" in attr: + needs_picklist_flush = True + req = _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + await self._execute_raw(req) + created.append(column_name) + + if needs_picklist_flush: + self._flush_cache("picklist") + + return created + + async def _delete_columns( + self, + table_schema_name: str, + columns: Union[str, List[str]], + ) -> List[str]: + """Delete one or more columns from a table. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param columns: Single column name or list of column names + :type columns: ``str`` | ``list[str]`` + + :return: List of deleted column schema names (empty if none removed). + :rtype: ``list[str]`` + + :raises TypeError: If ``columns`` is neither a ``str`` nor ``list[str]``. + :raises ValueError: If any provided column name is empty. + :raises MetadataError: If the table or a specified column does not exist. + :raises RuntimeError: If column metadata lacks a required ``MetadataId``. + :raises HttpError: If an underlying delete request fails. + """ + if isinstance(columns, str): + names = [columns] + elif isinstance(columns, list): + names = columns + else: + raise TypeError("columns must be str or list[str]") + + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ValueError("column names must be non-empty strings") + + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + # Use the actual SchemaName from the entity metadata + entity_schema = ent.get("SchemaName") or table_schema_name + metadata_id = ent.get("MetadataId") + deleted: List[str] = [] + needs_picklist_flush = False + + attr_metas = [ + await self._get_attribute_metadata(metadata_id, col, extra_select="@odata.type,AttributeType") + for col in names + ] + for column_name, attr_meta in zip(names, attr_metas): + if not attr_meta: + raise MetadataError( + f"Column '{column_name}' not found on table '{entity_schema}'.", + subcode=METADATA_COLUMN_NOT_FOUND, + ) + + attr_metadata_id = attr_meta.get("MetadataId") + if not attr_metadata_id: + raise RuntimeError(f"Metadata incomplete for column '{column_name}' (missing MetadataId).") + + await self._execute_raw(self._build_delete_column(metadata_id, attr_metadata_id)) + + attr_type = attr_meta.get("@odata.type") or attr_meta.get("AttributeType") + if isinstance(attr_type, str): + attr_type_l = attr_type.lower() + if "picklist" in attr_type_l or "optionset" in attr_type_l: + needs_picklist_flush = True + + deleted.append(column_name) + + if needs_picklist_flush: + self._flush_cache("picklist") + + return deleted + + # ---------------------- _build_* methods (no HTTP, but may call async helpers) --------------- + + async def _build_create( + self, + entity_set: str, + table: str, + data: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record POST request without sending it.""" + body = self._lowercase_keys(data) + body = await self._convert_labels_to_ints(table, body) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}", + body=json.dumps(body, ensure_ascii=False), + content_id=content_id, + ) + + async def _build_create_multiple( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build a CreateMultiple POST request without sending it.""" + if not all(isinstance(r, dict) for r in records): + raise TypeError("All items for multi-create must be dicts") + logical_name = table.lower() + lowered = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered] + enriched = [ + {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} if "@odata.type" not in r else r + for r in converted + ] + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.CreateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + async def _build_update( + self, + table: str, + record_id: str, + changes: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record PATCH request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference; in that case the + URL is the reference itself (resolved server-side within a changeset). + """ + body = self._lowercase_keys(changes) + body = await self._convert_labels_to_ints(table, body) + if record_id.startswith("$"): + url = record_id + else: + entity_set = await self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + headers={"If-Match": "*"}, + content_id=content_id, + ) + + async def _build_update_multiple_from_records( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request from pre-assembled records. + + Each record must already contain the primary key attribute. This helper + is shared by :meth:`_update_multiple` (which pre-assembles records) and + :meth:`_build_update_multiple` (which assembles from ids + changes). + """ + logical_name = table.lower() + lowered = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered] + enriched = [ + {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} if "@odata.type" not in r else r + for r in converted + ] + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + async def _build_update_multiple( + self, + entity_set: str, + table: str, + ids: List[str], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request without sending it.""" + pk_attr = await self._primary_id_attr(table) + if isinstance(changes, dict): + records = [{pk_attr: rid, **changes} for rid in ids] + elif isinstance(changes, list): + if len(changes) != len(ids): + raise ValidationError( + "ids and changes lists must have equal length for paired update.", + subcode="ids_changes_length_mismatch", + ) + records = [{pk_attr: rid, **ch} for rid, ch in zip(ids, changes)] + else: + raise ValidationError("changes must be a dict or list[dict].", subcode="invalid_changes_type") + return await self._build_update_multiple_from_records(entity_set, table, records) + + async def _build_upsert( + self, + entity_set: str, + table: str, + alternate_key: Dict[str, Any], + record: Dict[str, Any], + ) -> _RawRequest: + """Build a single-record PATCH upsert request without sending it. + + Unlike :meth:`_build_update`, no ``If-Match: *`` header is added so the + server creates the record when it does not yet exist. + """ + body = self._lowercase_keys(record) + body = await self._convert_labels_to_ints(table, body) + key_str = self._build_alternate_key_str(alternate_key) + url = f"{self.api}/{entity_set}({key_str})" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + async def _build_upsert_multiple( + self, + entity_set: str, + table: str, + alternate_keys: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpsertMultiple POST request without sending it.""" + if len(alternate_keys) != len(records): + raise ValidationError( + f"alternate_keys and records must have the same length " f"({len(alternate_keys)} != {len(records)})", + subcode="upsert_length_mismatch", + ) + logical_name = table.lower() + lowered_records = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered_records] + targets: List[Dict[str, Any]] = [] + for alt_key, record_processed in zip(alternate_keys, converted): + alt_key_lower = self._lowercase_keys(alt_key) + conflicting = { + k for k in set(alt_key_lower) & set(record_processed) if alt_key_lower[k] != record_processed[k] + } + if conflicting: + raise ValidationError( + f"record payload conflicts with alternate_key on fields: {sorted(conflicting)!r}", + subcode="upsert_key_conflict", + ) + if "@odata.type" not in record_processed: + record_processed["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" + key_str = self._build_alternate_key_str(alt_key) + record_processed["@odata.id"] = f"{entity_set}({key_str})" + targets.append(record_processed) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpsertMultiple", + body=json.dumps({"Targets": targets}, ensure_ascii=False), + ) + + async def _build_delete( + self, + table: str, + record_id: str, + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record DELETE request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference. + """ + if record_id.startswith("$"): + url = record_id + else: + entity_set = await self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="DELETE", + url=url, + headers={"If-Match": "*"}, + content_id=content_id, + ) + + async def _build_delete_multiple(self, table: str, ids: List[str]) -> _RawRequest: + """Build a BulkDelete POST request without sending it.""" + pk_attr = await self._primary_id_attr(table) + logical_name = table.lower() + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + payload = { + "JobName": f"Bulk delete {table} records @ {timestamp}", + "SendEmailNotification": False, + "ToRecipients": [], + "CCRecipients": [], + "RecurrencePattern": "", + "StartDateTime": timestamp, + "QuerySet": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.QueryExpression", + "EntityName": logical_name, + "ColumnSet": { + "@odata.type": "Microsoft.Dynamics.CRM.ColumnSet", + "AllColumns": False, + "Columns": [], + }, + "Criteria": { + "@odata.type": "Microsoft.Dynamics.CRM.FilterExpression", + "FilterOperator": "And", + "Conditions": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.ConditionExpression", + "AttributeName": pk_attr, + "Operator": "In", + "Values": [{"Value": rid, "Type": "System.Guid"} for rid in ids], + } + ], + }, + } + ], + } + return _RawRequest( + method="POST", + url=f"{self.api}/BulkDelete", + body=json.dumps(payload, ensure_ascii=False), + ) + + async def _build_get( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> _RawRequest: + """Build a single-record GET request without sending it.""" + entity_set = await self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if expand: + params.append("$expand=" + ",".join(expand)) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if params: + url += "?" + "&".join(params) + headers = None + if include_annotations: + headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} + return _RawRequest(method="GET", url=url, headers=headers) + + async def _build_list( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> _RawRequest: + """Build a multi-record GET request (single page, no pagination) without sending it.""" + entity_set = await self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if filter: + params.append("$filter=" + filter) + if orderby: + params.append("$orderby=" + ",".join(orderby)) + if top is not None: + params.append(f"$top={top}") + if expand: + params.append("$expand=" + ",".join(expand)) + if count: + params.append("$count=true") + url = f"{self.api}/{entity_set}" + if params: + url += "?" + "&".join(params) + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None + return _RawRequest(method="GET", url=url, headers=headers) + + async def _build_sql(self, sql: str) -> _RawRequest: + """Build a SQL query GET request without sending it. + + Resolves the entity set from the table name in the SQL statement via + :meth:`_extract_logical_table`, then embeds the SQL as a URL-encoded + ``?sql=`` query parameter. + + Uses ``urllib.parse.quote`` (``%20`` for spaces) rather than + ``urllib.parse.urlencode`` (``+`` for spaces). Both are accepted by + Dataverse and ``%20`` is the canonical RFC 3986 encoding for query- + string values. + + :param sql: SELECT statement (non-empty string; caller is responsible + for validation). + """ + logical = self._extract_logical_table(sql) + entity_set = await self._entity_set_from_schema_name(logical) + return _RawRequest( + method="GET", + url=f"{self.api}/{entity_set}?sql={_url_quote(sql, safe='')}", + ) diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py b/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py new file mode 100644 index 00000000..cbeac29a --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py @@ -0,0 +1,263 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async relationship metadata operations for Dataverse Web API. + +This module provides mixin functionality for relationship CRUD operations. +""" + +from __future__ import annotations + +__all__ = [] + +import asyncio +import re +from typing import Any, Dict, List, Optional + +from ...core.errors import MetadataError +from ...core._error_codes import METADATA_TABLE_NOT_FOUND + + +class _AsyncRelationshipOperationsMixin: + """ + Async mixin providing relationship metadata operations. + + This mixin is designed to be used with _AsyncODataClient and depends on: + - self.api: The API base URL + - self._headers(): Method to get auth headers + - self._request(): Async method to make HTTP requests + """ + + async def _create_one_to_many_relationship( + self, + lookup, + relationship, + solution: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Create a one-to-many relationship with lookup attribute. + + Posts to /RelationshipDefinitions with OneToManyRelationshipMetadata. + + :param lookup: Lookup attribute metadata (LookupAttributeMetadata instance). + :type lookup: ~PowerPlatform.Dataverse.models.relationship.LookupAttributeMetadata + :param relationship: Relationship metadata (OneToManyRelationshipMetadata instance). + :type relationship: ~PowerPlatform.Dataverse.models.relationship.OneToManyRelationshipMetadata + :param solution: Optional solution unique name to add the relationship to. + :type solution: ``str`` | ``None`` + + :return: Dictionary with relationship_id, attribute_id, and schema names. + :rtype: ``dict[str, Any]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + + # Build the payload by combining relationship and lookup metadata + payload = relationship.to_dict() + payload["Lookup"] = lookup.to_dict() + + headers = (await self._headers()).copy() + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + + r = await self._request("post", url, headers=headers, json=payload) + + # Extract IDs from response headers + relationship_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "relationship_id": relationship_id, + "relationship_schema_name": relationship.schema_name, + "lookup_schema_name": lookup.schema_name, + "referenced_entity": relationship.referenced_entity, + "referencing_entity": relationship.referencing_entity, + } + + async def _create_many_to_many_relationship( + self, + relationship, + solution: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Create a many-to-many relationship. + + Posts to /RelationshipDefinitions with ManyToManyRelationshipMetadata. + + :param relationship: Relationship metadata (ManyToManyRelationshipMetadata instance). + :type relationship: ~PowerPlatform.Dataverse.models.relationship.ManyToManyRelationshipMetadata + :param solution: Optional solution unique name to add the relationship to. + :type solution: ``str`` | ``None`` + + :return: Dictionary with relationship_id and schema name. + :rtype: ``dict[str, Any]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + + payload = relationship.to_dict() + + headers = (await self._headers()).copy() + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + + r = await self._request("post", url, headers=headers, json=payload) + + # Extract ID from response header + relationship_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "relationship_id": relationship_id, + "relationship_schema_name": relationship.schema_name, + "entity1_logical_name": relationship.entity1_logical_name, + "entity2_logical_name": relationship.entity2_logical_name, + } + + async def _delete_relationship(self, relationship_id: str) -> None: + """ + Delete a relationship by its metadata ID. + + :param relationship_id: The GUID of the relationship metadata. + :type relationship_id: ``str`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions({relationship_id})" + headers = (await self._headers()).copy() + headers["If-Match"] = "*" + await self._request("delete", url, headers=headers) + + async def _get_relationship(self, schema_name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve relationship metadata by schema name. + + :param schema_name: The schema name of the relationship. + :type schema_name: ``str`` + + :return: Relationship metadata dictionary, or None if not found. + :rtype: ``dict[str, Any]`` | ``None`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + params = {"$filter": f"SchemaName eq '{self._escape_odata_quotes(schema_name)}'"} + r = await self._request("get", url, headers=await self._headers(), params=params) + data = r.json() + results = data.get("value", []) + return results[0] if results else None + + async def _list_relationships( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationship definitions. + + Issues ``GET /RelationshipDefinitions`` with optional ``$filter`` and + ``$select`` query parameters. + + :param filter: Optional OData ``$filter`` expression. For example, + ``"RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'"`` + returns only one-to-many relationships. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + + :return: List of raw relationship metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + params: Dict[str, str] = {} + if filter: + params["$filter"] = filter + if select: + params["$select"] = ",".join(select) + r = await self._request("get", url, headers=await self._headers(), params=params) + return (r.json()).get("value", []) + + async def _list_table_relationships( + self, + table_schema_name: str, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationships for a specific table. + + Issues ``GET EntityDefinitions({MetadataId})/OneToManyRelationships``, + ``GET EntityDefinitions({MetadataId})/ManyToOneRelationships``, and + ``GET EntityDefinitions({MetadataId})/ManyToManyRelationships``, + then combines the results. + + :param table_schema_name: Schema name of the table (e.g. ``"account"``). + :type table_schema_name: ``str`` + :param filter: Optional OData ``$filter`` expression applied to each + sub-request. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + + :return: Combined list of one-to-many, many-to-one, and many-to-many + relationship metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table is not found. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + metadata_id = ent["MetadataId"] + # OneToMany/ManyToOne share the same property surface (ReferencedEntity, + # ReferencingEntity, etc.). ManyToManyRelationshipMetadata has a + # different schema -- it only exposes SchemaName plus Entity1/Entity2 + # fields, not ReferencedEntity or ReferencingEntity. Sending a $select + # that includes those properties to the ManyToMany endpoint causes a + # 400: "Could not find a property named 'ReferencedEntity' on type + # 'ManyToManyRelationshipMetadata'". Use separate param dicts. + one_to_many_params: Dict[str, str] = {} + many_to_many_params: Dict[str, str] = {} + if filter: + one_to_many_params["$filter"] = filter + many_to_many_params["$filter"] = filter + if select: + one_to_many_params["$select"] = ",".join(select) + + one_to_many_url = f"{self.api}/EntityDefinitions({metadata_id})/OneToManyRelationships" + many_to_one_url = f"{self.api}/EntityDefinitions({metadata_id})/ManyToOneRelationships" + many_to_many_url = f"{self.api}/EntityDefinitions({metadata_id})/ManyToManyRelationships" + + headers = await self._headers() + r1, r2, r3 = await asyncio.gather( + self._request("get", one_to_many_url, headers=headers, params=one_to_many_params), + self._request("get", many_to_one_url, headers=headers, params=one_to_many_params), + self._request("get", many_to_many_url, headers=headers, params=many_to_many_params), + ) + + return r1.json().get("value", []) + r2.json().get("value", []) + r3.json().get("value", []) + + def _extract_id_from_header(self, header_value: Optional[str]) -> Optional[str]: + """ + Extract a GUID from an OData-EntityId header value. + + :param header_value: The header value containing a URL with GUID. + :type header_value: ``str`` | ``None`` + + :return: Extracted GUID or None if not found. + :rtype: ``str`` | ``None`` + """ + if not header_value: + return None + match = re.search(r"\(([0-9a-fA-F-]+)\)", header_value) + return match.group(1) if match else None diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_upload.py b/src/PowerPlatform/Dataverse/aio/data/_async_upload.py new file mode 100644 index 00000000..59c36967 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_upload.py @@ -0,0 +1,193 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async file upload helpers.""" + +from __future__ import annotations + +import asyncio +import math +from pathlib import Path +from typing import Optional +from urllib.parse import quote + + +class _AsyncFileUploadMixin: + """Async file upload capabilities (small + chunk) with auto selection.""" + + async def _upload_file( + self, + table_schema_name: str, + record_id: str, + file_name_attribute: str, + path: str, + mode: Optional[str] = None, + mime_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file to a Dataverse file column with automatic method selection. + + Parameters + ---------- + table_schema_name : :class:`str` + Table schema name, e.g. "account" or "new_MyTestTable". + record_id : :class:`str` + GUID of the target record. + file_name_attribute : :class:`str` + Schema name of the file column attribute (e.g., "new_Document"). If the column doesn't exist, it will be created. + path : :class:`str` + Local filesystem path to the file. + mode : :class:`str` | None + Upload strategy: "auto" (default), "small", or "chunk". + mime_type : :class:`str` | None + Explicit MIME type. If omitted falls back to application/octet-stream. + if_none_match : :class:`bool` + When True (default) only succeeds if column empty. When False overwrites (If-Match: *). + """ + # Resolve entity set from table schema name + entity_set = await self._entity_set_from_schema_name(table_schema_name) + + # Check if the file column exists, create it if it doesn't + entity_metadata = await self._get_entity_by_table_schema_name(table_schema_name) + if entity_metadata: + metadata_id = entity_metadata.get("MetadataId") + if metadata_id: + attr_metadata = await self._get_attribute_metadata(metadata_id, file_name_attribute) + if not attr_metadata: + # Attribute doesn't exist, create it + await self._create_columns(table_schema_name, {file_name_attribute: "file"}) + # Wait for the attribute to become visible in the data API + # Raises RuntimeError with underlying exception if timeout occurs + await self._wait_for_attribute_visibility(entity_set, file_name_attribute) + + mode = (mode or "auto").lower() + + if mode == "auto": + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + size = p.stat().st_size + mode = "small" if size < 128 * 1024 * 1024 else "chunk" + + # Convert schema name to lowercase logical name for URL usage + logical_name = file_name_attribute.lower() + + if mode == "small": + return await self._upload_file_small( + entity_set, record_id, logical_name, path, content_type=mime_type, if_none_match=if_none_match + ) + if mode == "chunk": + return await self._upload_file_chunk(entity_set, record_id, logical_name, path, if_none_match=if_none_match) + raise ValueError(f"Invalid mode '{mode}'. Use 'auto', 'small', or 'chunk'.") + + async def _upload_file_small( + self, + entity_set: str, + record_id: str, + file_name_attribute: str, + path: str, + content_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file (<128MB) via single PATCH.""" + if not record_id: + raise ValueError("record_id required") + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + size = p.stat().st_size + limit = 128 * 1024 * 1024 + if size > limit: + raise ValueError(f"File size {size} exceeds single-upload limit {limit}; use chunk mode.") + data = await asyncio.to_thread(p.read_bytes) + fname = p.name + key = self._format_key(record_id) + url = f"{self.api}/{entity_set}{key}/{file_name_attribute}" + headers = { + "Content-Type": content_type or "application/octet-stream", + "x-ms-file-name": fname, + } + if if_none_match: + headers["If-None-Match"] = "null" + else: + headers["If-Match"] = "*" + # Single PATCH upload; allow default success codes (includes 204) + await self._request("patch", url, headers=headers, data=data) + + async def _upload_file_chunk( + self, + entity_set: str, + record_id: str, + file_name_attribute: str, + path: str, + if_none_match: bool = True, + ) -> None: + """Stream a local file using Dataverse native chunked PATCH protocol. + 1. Initial PATCH with header x-ms-transfer-mode: chunked (empty body) to start session. + 2. Subsequent PATCH calls to Location URL including sessiontoken with binary body segments and headers. Returns 206 for partial chunks and 204 on final. + + Parameters + ---------- + entity_set : :class:`str` + Target entity set (plural logical name), e.g. "accounts". + record_id : :class:`str` + GUID of the target record. + file_name_attribute : :class:`str` + Logical name of the file column attribute. + path : :class:`str` + Local filesystem path to the file. + if_none_match : :class:`bool` + When True sends ``If-None-Match: null`` to only succeed if the column is currently empty. + Set False to always overwrite (uses ``If-Match: *``). + + Returns + ------- + None + Returns nothing on success. Any failure raises an exception. + """ + if not record_id: + raise ValueError("record_id required") + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + total_size = p.stat().st_size + fname = p.name + key = self._format_key(record_id) + init_url = f"{self.api}/{entity_set}{key}/{file_name_attribute}?x-ms-file-name={quote(fname)}" + headers = { + "x-ms-transfer-mode": "chunked", + } + if if_none_match: + headers["If-None-Match"] = "null" + else: + headers["If-Match"] = "*" + r_init = await self._request("patch", init_url, headers=headers, data=b"") + location = r_init.headers.get("Location") or r_init.headers.get("location") + if not location: + raise RuntimeError("Missing Location header with sessiontoken for chunked upload") + rec_hdr = r_init.headers.get("x-ms-chunk-size") or r_init.headers.get("X-MS-CHUNK-SIZE") + try: + recommended_size = int(rec_hdr) if rec_hdr else None + except Exception: # noqa: BLE001 + recommended_size = None + effective_size = recommended_size or (4 * 1024 * 1024) + if effective_size <= 0: + raise ValueError("effective chunk size must be positive") + total_chunks = int(math.ceil(total_size / effective_size)) if total_size else 1 + uploaded_bytes = 0 + with p.open("rb") as fh: + for _ in range(total_chunks): + chunk = await asyncio.to_thread(fh.read, effective_size) + if not chunk: + break + start = uploaded_bytes + end = start + len(chunk) - 1 + c_headers = { + "x-ms-file-name": fname, + "Content-Type": "application/octet-stream", + "Content-Range": f"bytes {start}-{end}/{total_size}", + "Content-Length": str(len(chunk)), + } + # Each chunk returns 206 (partial) or 204 (final). Accept both. + await self._request("patch", location, headers=c_headers, data=chunk, expected=(206, 204)) + uploaded_bytes += len(chunk) diff --git a/src/PowerPlatform/Dataverse/aio/models/__init__.py b/src/PowerPlatform/Dataverse/aio/models/__init__.py new file mode 100644 index 00000000..f4c2e3d0 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async data models and type definitions for the Dataverse SDK. + +Provides async-specific models for Dataverse entities: + +- :class:`~PowerPlatform.Dataverse.aio.models.async_query_builder.AsyncQueryBuilder`: Async fluent query builder. +- :class:`~PowerPlatform.Dataverse.aio.models.async_fetchxml_query.AsyncFetchXmlQuery`: Async FetchXML query. +""" + +__all__ = [] diff --git a/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py b/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py new file mode 100644 index 00000000..c10ca6f6 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""AsyncFetchXmlQuery — inert async query object returned by AsyncQueryOperations.fetchxml().""" + +from __future__ import annotations + +import warnings +import xml.etree.ElementTree as _ET +from typing import AsyncIterator, List, TYPE_CHECKING +from urllib.parse import unquote as _url_unquote, quote as _url_quote + +from ...core.errors import ValidationError +from ...models.fetchxml_query import _MAX_URL_LENGTH, _MAX_PAGES, _PREFER_HEADER +from ...models.record import QueryResult, Record + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncFetchXmlQuery"] + + +class AsyncFetchXmlQuery: + """Inert async FetchXML query object. No HTTP request is made until + :meth:`execute` or :meth:`execute_pages` is called. + + Obtained via ``client.query.fetchxml(xml)``. + + :param xml: Stripped, well-formed FetchXML string. + :param entity_name: Entity schema name from the ```` element. + :param client: Parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient`. + """ + + def __init__(self, xml: str, entity_name: str, client: "AsyncDataverseClient") -> None: + self._xml = xml + self._entity_name = entity_name + self._client = client + + async def execute(self) -> QueryResult: + """Execute the FetchXML query and return all results as a :class:`QueryResult`. + + Awaitable — fetches all pages and holds every record in memory before + returning. Use :meth:`execute_pages` when the result set may be large. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + rows = await client.query.fetchxml(xml).execute() + df = rows.to_dataframe() + """ + all_records: List[Record] = [] + async for page in self.execute_pages(): + all_records.extend(page.records) + return QueryResult(all_records) + + async def execute_pages(self) -> AsyncIterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Each iteration fires one HTTP request and yields one page. One-shot — + do not iterate more than once. + + :return: Async iterator of per-page :class:`QueryResult` objects. + :rtype: AsyncIterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + async for page in client.query.fetchxml(xml).execute_pages(): + process(page.to_dataframe()) + """ + current_xml = self._xml + page_num = 1 + page_count = 0 + + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(self._entity_name) + base_url = f"{od.api}/{entity_set}" + + while True: + page_count += 1 + if page_count > _MAX_PAGES: + raise ValidationError( + f"FetchXML paging exceeded {_MAX_PAGES} pages. " + "This may indicate a runaway query or a bug in paging cookie propagation." + ) + + encoded_len = len(base_url) + len("?fetchXml=") + len(_url_quote(current_xml, safe="")) + if encoded_len > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML request URL exceeds {_MAX_URL_LENGTH} characters after encoding. " + "Simplify the query or reduce attributes/conditions." + ) + + r = await od._request( + "get", + base_url, + headers={"Prefer": _PREFER_HEADER}, + params={"fetchXml": current_xml}, + ) + try: + data = r.json() + except Exception: + data = {} + + items = data.get("value") if isinstance(data, dict) else None + page_records: List[Record] = [] + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + page_records.append(Record.from_api_response(self._entity_name, item)) + + yield QueryResult(page_records) + + more_raw = data.get("@Microsoft.Dynamics.CRM.morerecords", False) if isinstance(data, dict) else False + more = more_raw is True or (isinstance(more_raw, str) and more_raw.lower() == "true") + if not more: + break + + raw_cookie = ( + data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" + ) + + _cookie_parse_error = False + if raw_cookie: + try: + cookie_el = _ET.fromstring(raw_cookie) + inner_encoded = cookie_el.get("pagingcookie", "") + if inner_encoded: + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") + continue + except (_ET.ParseError, ValueError) as exc: + warnings.warn( + f"FetchXML paging cookie could not be parsed ({exc}); " "falling back to simple paging.", + UserWarning, + stacklevel=2, + ) + _cookie_parse_error = True + + if not _cookie_parse_error: + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) + page_num += 1 + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py b/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py new file mode 100644 index 00000000..e48b3198 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""AsyncQueryBuilder — async execution layer over the shared QueryBuilder.""" + +from __future__ import annotations + +from typing import AsyncIterator, List + +from ...models.query_builder import _QueryBuilderBase +from ...models.record import QueryResult, Record + +__all__ = ["AsyncQueryBuilder"] + + +class AsyncQueryBuilder(_QueryBuilderBase): + """Async-capable QueryBuilder. + + Identical fluent interface to :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder` + — all chaining methods (``select``, ``where``, ``order_by``, ``top``, ``page_size``, + ``count``, ``expand``, ``include_annotations``, ``include_formatted_values``) are + inherited unchanged. Only the execution methods are overridden as coroutines. + + Obtained via ``client.query.builder(table)`` on an async client. + + Example:: + + from PowerPlatform.Dataverse.models.filters import col + + result = await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()) + for record in result: + print(record["name"]) + """ + + async def execute(self) -> QueryResult: + """Execute the query and return all results as a :class:`QueryResult`. + + Awaitable — fetches all pages and holds every record in memory before + returning. Use :meth:`execute_pages` for lazy per-page streaming. + + At least one of ``select()``, ``where()``, ``top()``, or + ``page_size()`` must be called first to prevent accidental full-table + scans. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + :raises ValueError: If no scope constraint has been set. + :raises RuntimeError: If the builder was not created via + ``client.query.builder()``. + + Example:: + + result = await (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute()) + for record in result: + print(record["name"]) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute() to prevent accidental full-table scans." + ) + params = self.build() + client = self._query_ops._client + all_records: List[Record] = [] + async with client._scoped_odata() as od: + async for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + all_records.extend(Record.from_api_response(params["table"], row) for row in page) + return QueryResult(all_records) + + async def execute_pages(self) -> AsyncIterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Each iteration triggers one network request. One-shot — do not + iterate more than once. + + At least one of ``select()``, ``where()``, ``top()``, or + ``page_size()`` must be called first to prevent accidental full-table + scans. + + :return: Async iterator of per-page + :class:`~PowerPlatform.Dataverse.models.record.QueryResult` objects. + :rtype: AsyncIterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + :raises ValueError: If no scope constraint has been set. + :raises RuntimeError: If the builder was not created via + ``client.query.builder()``. + + Example:: + + async for page in (client.query.builder("account") + .select("name") + .execute_pages()): + process(page.to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute_pages() to prevent accidental full-table scans." + ) + params = self.build() + client = self._query_ops._client + async with client._scoped_odata() as od: + async for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + yield QueryResult([Record.from_api_response(params["table"], row) for row in page]) diff --git a/src/PowerPlatform/Dataverse/aio/operations/__init__.py b/src/PowerPlatform/Dataverse/aio/operations/__init__.py new file mode 100644 index 00000000..62bb4281 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async operation namespaces for the Dataverse SDK. + +This module contains the async operation namespace classes that organize +SDK operations into logical groups: records, query, tables, files, and batch. +""" + +from typing import List + +__all__: List[str] = [] diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_batch.py b/src/PowerPlatform/Dataverse/aio/operations/async_batch.py new file mode 100644 index 00000000..3b95fea3 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_batch.py @@ -0,0 +1,174 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async batch operation namespaces for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List + +from ...data._batch_base import _ChangeSet +from ...operations.batch import ( + BatchDataFrameOperations, + BatchQueryOperations, + BatchRecordOperations, + BatchTableOperations, + ChangeSetRecordOperations, +) +from ..data._async_batch import _AsyncBatchClient +from ...models.batch import BatchResult + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + +__all__ = [ + "AsyncBatchRequest", + "AsyncBatchOperations", + "AsyncChangeSet", +] + + +# --------------------------------------------------------------------------- +# Changeset +# --------------------------------------------------------------------------- + + +class AsyncChangeSet: + """ + A transactional group of single-record write operations. + + All operations succeed or are rolled back together. Use as an async context + manager or call :attr:`records` to add operations directly. + + Do not instantiate directly; use :meth:`AsyncBatchRequest.changeset`. + + Example:: + + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + """ + + def __init__(self, internal: _ChangeSet) -> None: + self._internal = internal + self.records = ChangeSetRecordOperations(internal) + + async def __aenter__(self) -> "AsyncChangeSet": + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + return None + + +# --------------------------------------------------------------------------- +# AsyncBatchRequest and AsyncBatchOperations +# --------------------------------------------------------------------------- + + +class AsyncBatchRequest: + """ + Builder for constructing and executing a Dataverse OData ``$batch`` request. + + Obtain via :meth:`AsyncBatchOperations.new` (``client.batch.new()``). Add operations + through :attr:`records`, :attr:`tables`, :attr:`query`, and :attr:`dataframe`, + optionally group writes into a :meth:`changeset`, then call :meth:`execute`. + + Operations are executed sequentially in the order added. The resulting + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` contains one + :class:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse` per HTTP + request dispatched (some operations expand to multiple requests). + + .. note:: + Maximum 1000 HTTP operations per batch. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Contoso"}) + batch.tables.get("account") + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + result = await batch.execute() + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + self._items: List[Any] = [] + self._content_id_counter: List[int] = [1] # shared across all changesets + self.records = BatchRecordOperations(self) + self.tables = BatchTableOperations(self) + self.query = BatchQueryOperations(self) + self.dataframe = BatchDataFrameOperations(self) + + def changeset(self) -> AsyncChangeSet: + """ + Create a new :class:`AsyncChangeSet` attached to this batch. + + The changeset is added to the batch immediately. Operations added to + the returned :class:`AsyncChangeSet` via ``cs.records.*`` execute atomically. + + :returns: A new :class:`AsyncChangeSet` ready to receive operations. + + Example:: + + async with batch.changeset() as cs: + cs.records.create("account", {"name": "ACME"}) + cs.records.create("contact", {"firstname": "Bob"}) + """ + internal = _ChangeSet(_counter=self._content_id_counter) + self._items.append(internal) + return AsyncChangeSet(internal) + + async def execute(self, *, continue_on_error: bool = False) -> BatchResult: + """ + Submit the batch to Dataverse and return all responses. + + :param continue_on_error: When False (default), Dataverse stops at the + first failure and returns that operation's error as a 4xx response. + When True, ``Prefer: odata.continue-on-error`` is sent and all + operations are attempted. + :returns: :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` + with one entry per HTTP operation in submission order. + :raises ValidationError: If the batch exceeds 1000 operations or an + unsupported column type is specified. + :raises MetadataError: If metadata pre-resolution fails (table or + column not found) for ``tables.delete``, ``tables.add_columns``, + or ``tables.remove_columns``. + :raises HttpError: On HTTP-level failures (auth, server error, etc.) + that prevent the batch from executing. + """ + async with self._client._scoped_odata() as od: + return await _AsyncBatchClient(od).execute(self._items, continue_on_error=continue_on_error) + + +class AsyncBatchOperations: + """ + Async namespace for batch operations (``client.batch``). + + Accessed via ``client.batch``. Use :meth:`new` to create an + :class:`AsyncBatchRequest` builder. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Fabrikam"}) + result = await batch.execute() + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + def new(self) -> AsyncBatchRequest: + """ + Create a new empty :class:`AsyncBatchRequest` builder. + + :returns: An empty :class:`AsyncBatchRequest`. + """ + return AsyncBatchRequest(self._client) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py b/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py new file mode 100644 index 00000000..fba7b334 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py @@ -0,0 +1,309 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async DataFrame CRUD operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import pandas as pd + +from ...utils._pandas import dataframe_to_records + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncDataFrameOperations"] + + +class AsyncDataFrameOperations: + """Async namespace for pandas DataFrame CRUD operations. + + Accessed via ``client.dataframe``. Provides DataFrame-oriented wrappers + around the async record-level CRUD operations. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + import pandas as pd + + async with AsyncDataverseClient(base_url, credential) as client: + + # Query records as a DataFrame via SQL + df = await client.dataframe.sql( + "SELECT TOP 100 name FROM account WHERE statecode = 0" + ) + + # Create records from a DataFrame + new_df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + new_df["accountid"] = await client.dataframe.create("account", new_df) + + # Update records + new_df["telephone1"] = ["555-0100", "555-0200"] + await client.dataframe.update("account", new_df, id_column="accountid") + + # Delete records + await client.dataframe.delete("account", new_df["accountid"]) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # --------------------------------------------------------------------- sql + + async def sql(self, sql: str) -> pd.DataFrame: + """Execute a SQL query and return the results as a pandas DataFrame. + + Delegates to :meth:`~PowerPlatform.Dataverse.aio.operations.async_query.AsyncQueryOperations.sql` + and converts the list of records into a single DataFrame. + + :param sql: Supported SQL SELECT statement. + :type sql: :class:`str` + + :return: DataFrame containing all result rows. Returns an empty + DataFrame when no rows match. + :rtype: ~pandas.DataFrame + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``sql`` is not a string or is empty. + + Example: + SQL query to DataFrame:: + + df = await client.dataframe.sql( + "SELECT TOP 100 name, revenue FROM account " + "WHERE statecode = 0 ORDER BY revenue" + ) + print(f"Got {len(df)} rows") + print(df.head()) + + Aggregate query to DataFrame:: + + df = await client.dataframe.sql( + "SELECT a.name, COUNT(c.contactid) as cnt " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "GROUP BY a.name" + ) + """ + rows = await self._client.query.sql(sql) + if not rows: + return pd.DataFrame() + return pd.DataFrame.from_records([r.data for r in rows]) + + # ----------------------------------------------------------------- create + + async def create( + self, + table: str, + records: pd.DataFrame, + ) -> pd.Series: + """Create records from a pandas DataFrame. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param records: DataFrame where each row is a record to create. + :type records: ~pandas.DataFrame + + :return: Series of created record GUIDs, aligned with the input DataFrame index. + :rtype: ~pandas.Series + + :raises TypeError: If ``records`` is not a pandas DataFrame. + :raises ValueError: If ``records`` is empty or the number of returned + IDs does not match the number of input rows. + + .. tip:: + All rows are sent in a single ``CreateMultiple`` request. For very + large DataFrames, consider splitting into smaller batches to avoid + request timeouts. + + Example: + Create records from a DataFrame:: + + import pandas as pd + + df = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ]) + df["accountid"] = await client.dataframe.create("account", df) + """ + if not isinstance(records, pd.DataFrame): + raise TypeError("records must be a pandas DataFrame") + + if records.empty: + raise ValueError("records must be a non-empty DataFrame") + + record_list = dataframe_to_records(records) + + # Detect rows where all values were NaN/None (empty dicts after normalization) + empty_rows = [records.index[i] for i, r in enumerate(record_list) if not r] + if empty_rows: + raise ValueError( + f"Records at index(es) {empty_rows} have no non-null values. " + "All rows must contain at least one field to create." + ) + + ids = await self._client.records.create(table, record_list) + + if len(ids) != len(records): + raise ValueError(f"Server returned {len(ids)} IDs for {len(records)} input rows") + + return pd.Series(ids, index=records.index) + + # ----------------------------------------------------------------- update + + async def update( + self, + table: str, + changes: pd.DataFrame, + id_column: str, + clear_nulls: bool = False, + ) -> None: + """Update records from a pandas DataFrame. + + Each row in the DataFrame represents an update. The ``id_column`` specifies which + column contains the record GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param changes: DataFrame where each row contains a record GUID and the fields to update. + :type changes: ~pandas.DataFrame + :param id_column: Name of the DataFrame column containing record GUIDs. + :type id_column: :class:`str` + :param clear_nulls: When ``False`` (default), missing values (NaN/None) are skipped + (the field is left unchanged on the server). When ``True``, missing values are sent + as ``null`` to Dataverse, clearing the field. Use ``True`` only when you intentionally + want NaN/None values to clear fields. + :type clear_nulls: :class:`bool` + + :raises TypeError: If ``changes`` is not a pandas DataFrame. + :raises ValueError: If ``changes`` is empty, ``id_column`` is not found in the + DataFrame, ``id_column`` contains invalid (non-string, empty, or whitespace-only) + values, or no updatable columns exist besides ``id_column``. + When ``clear_nulls`` is ``False`` (default), rows where all change values + are NaN/None produce empty patches and are silently skipped. If all rows + are skipped, the method returns without making an API call. When + ``clear_nulls`` is ``True``, NaN/None values become explicit nulls, so + rows are never skipped. + + .. tip:: + All rows are sent in a single ``UpdateMultiple`` request (or a + single PATCH for one row). For very large DataFrames, consider + splitting into smaller batches to avoid request timeouts. + + Example: + Update records with different values per row:: + + import pandas as pd + + df = pd.DataFrame([ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ]) + await client.dataframe.update("account", df, id_column="accountid") + + Broadcast the same change to all records:: + + df = pd.DataFrame({"accountid": ["guid-1", "guid-2", "guid-3"]}) + df["websiteurl"] = "https://example.com" + await client.dataframe.update("account", df, id_column="accountid") + + Clear a field by setting clear_nulls=True:: + + df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}]) + await client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + """ + if not isinstance(changes, pd.DataFrame): + raise TypeError("changes must be a pandas DataFrame") + if changes.empty: + raise ValueError("changes must be a non-empty DataFrame") + if id_column not in changes.columns: + raise ValueError(f"id_column '{id_column}' not found in DataFrame columns") + + raw_ids = changes[id_column].tolist() + invalid = [changes.index[i] for i, v in enumerate(raw_ids) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"id_column '{id_column}' contains invalid values at row index(es) {invalid}. " + "All IDs must be non-empty strings." + ) + ids = [v.strip() for v in raw_ids] + + change_columns = [column for column in changes.columns if column != id_column] + if not change_columns: + raise ValueError( + "No columns to update. The DataFrame must contain at least one column besides the id_column." + ) + change_list = dataframe_to_records(changes[change_columns], na_as_null=clear_nulls) + + # Filter out rows where all change values were NaN/None (empty dicts) + paired = [(rid, patch) for rid, patch in zip(ids, change_list) if patch] + if not paired: + return + ids_filtered: List[str] = [p[0] for p in paired] + change_filtered: List[Dict[str, Any]] = [p[1] for p in paired] + + if len(ids_filtered) == 1: + await self._client.records.update(table, ids_filtered[0], change_filtered[0]) + else: + await self._client.records.update(table, ids_filtered, change_filtered) + + # ----------------------------------------------------------------- delete + + async def delete( + self, + table: str, + ids: pd.Series, + use_bulk_delete: bool = True, + ) -> Optional[str]: + """Delete records by passing a pandas Series of GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param ids: Series of record GUIDs to delete. + :type ids: ~pandas.Series + :param use_bulk_delete: When ``True`` (default) and ``ids`` contains multiple values, + execute the BulkDelete action and return its async job identifier. + When ``False`` each record is deleted sequentially. + :type use_bulk_delete: :class:`bool` + + :raises TypeError: If ``ids`` is not a pandas Series. + :raises ValueError: If ``ids`` contains invalid (non-string, empty, or + whitespace-only) values. + + :return: BulkDelete job ID when deleting multiple records via BulkDelete; + ``None`` when deleting a single record, using sequential deletion, or + when ``ids`` is empty. + :rtype: :class:`str` or None + + Example: + Delete records using a Series:: + + import pandas as pd + + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + await client.dataframe.delete("account", ids) + """ + if not isinstance(ids, pd.Series): + raise TypeError("ids must be a pandas Series") + + raw_list = ids.tolist() + if not raw_list: + return None + + invalid = [ids.index[i] for i, v in enumerate(raw_list) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"ids Series contains invalid values at index(es) {invalid}. " f"All IDs must be non-empty strings." + ) + id_list = [v.strip() for v in raw_list] + + if len(id_list) == 1: + await self._client.records.delete(table, id_list[0]) + return None + return await self._client.records.delete(table, id_list, use_bulk_delete=use_bulk_delete) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_files.py b/src/PowerPlatform/Dataverse/aio/operations/async_files.py new file mode 100644 index 00000000..da755e62 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_files.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async file operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncFileOperations"] + + +class AsyncFileOperations: + """Async namespace for file operations. + + Accessed via ``client.files``. Provides file upload operations for + Dataverse file columns. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + await client.files.upload( + "account", account_id, "new_Document", "/path/to/file.pdf" + ) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- upload + + async def upload( + self, + table: str, + record_id: str, + file_column: str, + path: str, + *, + mode: Optional[str] = None, + mime_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file to a Dataverse file column. + + :param table: Schema name of the table (e.g. ``"account"`` or + ``"new_MyTestTable"``). + :type table: :class:`str` + :param record_id: GUID of the target record. + :type record_id: :class:`str` + :param file_column: Schema name of the file column attribute (e.g., + ``"new_Document"``). If the column doesn't exist, it will be + created automatically. + :type file_column: :class:`str` + :param path: Local filesystem path to the file. The stored filename + will be the basename of this path. + :type path: :class:`str` + :param mode: Upload strategy: ``"auto"`` (default), ``"small"``, or + ``"chunk"``. Auto mode selects small or chunked upload based on + file size. + :type mode: :class:`str` or None + :param mime_type: Explicit MIME type to store with the file (e.g. + ``"application/pdf"``). If not provided, defaults to + ``"application/octet-stream"``. + :type mime_type: :class:`str` or None + :param if_none_match: When True (default), sends + ``If-None-Match: null`` header to only succeed if the column is + currently empty. Set False to always overwrite using + ``If-Match: *``. + :type if_none_match: :class:`bool` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the upload fails or the file column is not empty when + ``if_none_match=True``. + :raises FileNotFoundError: If the specified file path does not exist. + + Example: + Upload a PDF file:: + + await client.files.upload( + "account", + account_id, + "new_Contract", + "/path/to/contract.pdf", + mime_type="application/pdf", + ) + + Upload with auto mode selection:: + + await client.files.upload( + "email", + email_id, + "new_Attachment", + "/path/to/large_file.zip", + ) + """ + async with self._client._scoped_odata() as od: + await od._upload_file( + table, + record_id, + file_column, + path, + mode=mode, + mime_type=mime_type, + if_none_match=if_none_match, + ) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_query.py b/src/PowerPlatform/Dataverse/aio/operations/async_query.py new file mode 100644 index 00000000..7f378380 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_query.py @@ -0,0 +1,371 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async query operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +import xml.etree.ElementTree as _ET +from typing import Any, Dict, List, TYPE_CHECKING +from urllib.parse import quote as _url_quote + +from ...core.errors import MetadataError, ValidationError +from ..models.async_fetchxml_query import AsyncFetchXmlQuery +from ..models.async_query_builder import AsyncQueryBuilder +from ...models.fetchxml_query import _MAX_URL_LENGTH +from ...models.record import Record + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncQueryOperations"] + + +class AsyncQueryOperations: + """Async namespace for query operations. + + Accessed via ``client.query``. Provides query and search operations + against Dataverse tables. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Fluent query builder (recommended) + from PowerPlatform.Dataverse.models.filters import col + + for record in await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()): + print(record["name"]) + + # SQL query + rows = await client.query.sql("SELECT TOP 10 name FROM account ORDER BY name") + for row in rows: + print(row["name"]) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- builder + + def builder(self, table: str) -> AsyncQueryBuilder: + """Create a fluent async query builder for the specified table. + + Returns an :class:`~PowerPlatform.Dataverse.models.async_query_builder.AsyncQueryBuilder` + that can be chained with filter, select, and order methods, then + executed via ``await .execute()`` or iterated via ``async for`` with + ``.execute_pages()``. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :return: An AsyncQueryBuilder instance bound to this client. + :rtype: ~PowerPlatform.Dataverse.models.async_query_builder.AsyncQueryBuilder + + Example:: + + from PowerPlatform.Dataverse.models.filters import col + + result = await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()) + for record in result: + print(record["name"]) + + # Lazy paged iteration + async for page in (client.query.builder("account") + .select("name") + .execute_pages()): + process(page.to_dataframe()) + """ + qb = AsyncQueryBuilder(table) + qb._query_ops = self + return qb + + # --------------------------------------------------------------- fetchxml + + def fetchxml(self, xml: str) -> AsyncFetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery` object. + + No HTTP request is made until + :meth:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery.execute` + or + :meth:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery.execute_pages` + is called on the returned object. + + :param xml: Well-formed FetchXML query string. The root ```` + element determines the entity set endpoint. + :type xml: :class:`str` + :return: Inert async query object. + :rtype: :class:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery` + :raises ValidationError: If the FetchXML is not a string, is empty, or exceeds the URL + length limit when encoded. + :raises ValueError: If the FetchXML is missing a root ```` element or name. + + Example:: + + query = client.query.fetchxml(\"\"\" + + + + + + \"\"\") + + # Eager — all pages collected: + result = await query.execute() + df = result.to_dataframe() + + # Lazy — one page at a time: + async for page in query.execute_pages(): + process(page.to_dataframe()) + """ + if not isinstance(xml, str): + raise ValidationError("xml must be a string") + xml = xml.strip() + if not xml: + raise ValidationError("xml must not be empty") + if len(_url_quote(xml, safe="")) > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML exceeds the Dataverse URL length limit ({_MAX_URL_LENGTH:,} characters) when encoded. " + "Use a $batch POST request to send FetchXML in the request body where the limit is 64 KB." + ) + try: + root_el = _ET.fromstring(xml) + except _ET.ParseError as exc: + raise ValidationError(f"xml is not well-formed: {exc}") from exc + entity_el = root_el.find("entity") + if entity_el is None: + raise ValueError("FetchXML must contain an child element") + entity_name = entity_el.get("name", "") + if not entity_name: + raise ValueError("FetchXML element must have a 'name' attribute") + return AsyncFetchXmlQuery(xml, entity_name, self._client) + + # -------------------------------------------------------------------- sql + + async def sql(self, sql: str) -> List[Record]: + """Execute a read-only SQL query using the Dataverse Web API. + + The Dataverse SQL endpoint supports a broad subset of T-SQL:: + + SELECT / SELECT DISTINCT / SELECT TOP N (0-5000) + FROM table [alias] + INNER JOIN / LEFT JOIN (multi-table, no depth limit) + WHERE (=, !=, >, <, >=, <=, LIKE, IN, NOT IN, IS NULL, + IS NOT NULL, BETWEEN, AND, OR, nested parentheses) + GROUP BY column + ORDER BY column [ASC|DESC] + OFFSET n ROWS FETCH NEXT m ROWS ONLY + COUNT(*), SUM(), AVG(), MIN(), MAX() + + ``SELECT *`` is not supported -- specify column names explicitly. + Use :meth:`sql_columns` to discover available column names for a table. + + Not supported: SELECT *, subqueries, CTE, HAVING, UNION, + RIGHT/FULL/CROSS JOIN, CASE, COALESCE, window functions, + string/date/math functions, INSERT/UPDATE/DELETE. For writes, use + ``client.records`` methods. + + :param sql: Supported SQL SELECT statement. + :type sql: :class:`str` + + :return: List of :class:`~PowerPlatform.Dataverse.models.record.Record` + objects. Returns an empty list when no rows match. + :rtype: list[~PowerPlatform.Dataverse.models.record.Record] + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``sql`` is not a string or is empty. + + Example: + Basic query:: + + rows = await client.query.sql( + "SELECT TOP 10 name FROM account ORDER BY name" + ) + + JOIN with aggregation:: + + rows = await client.query.sql( + "SELECT a.name, COUNT(c.contactid) as cnt " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "GROUP BY a.name" + ) + """ + async with self._client._scoped_odata() as od: + rows = await od._query_sql(sql) + return [Record.from_api_response("", row) for row in rows] + + # --------------------------------------------------------------- sql_columns + + async def sql_columns( + self, + table: str, + *, + include_system: bool = False, + ) -> List[Dict[str, Any]]: + """Return a simplified list of SQL-usable columns for a table. + + Each dict contains ``name`` (logical name for SQL), ``type`` + (Dataverse attribute type), ``is_pk`` (primary key flag), and + ``label`` (display name). Virtual columns are always excluded + because the SQL endpoint cannot query them. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param include_system: When ``False`` (default), columns that end + with common system suffixes (``_base``, ``versionnumber``, + ``timezoneruleversionnumber``, ``utcconversiontimezonecode``, + ``importsequencenumber``, ``overriddencreatedon``) are excluded. + :type include_system: :class:`bool` + + :return: List of column metadata dicts. + :rtype: list[dict[str, typing.Any]] + + Example:: + + cols = await client.query.sql_columns("account") + for c in cols: + print(f"{c['name']:30s} {c['type']:20s} PK={c['is_pk']}") + """ + _SYSTEM_SUFFIXES = ( + "_base", + "versionnumber", + "timezoneruleversionnumber", + "utcconversiontimezonecode", + "importsequencenumber", + "overriddencreatedon", + ) + + raw = await self._client.tables.list_columns( + table, + select=[ + "LogicalName", + "SchemaName", + "AttributeType", + "IsPrimaryId", + "IsPrimaryName", + "DisplayName", + "AttributeOf", + ], + filter="AttributeType ne 'Virtual'", + ) + result: List[Dict[str, Any]] = [] + for c in raw: + name = c.get("LogicalName", "") + if not name: + continue + if not include_system and any(name.endswith(s) for s in _SYSTEM_SUFFIXES): + continue + # Skip computed display-name columns (AttributeOf is set, meaning + # they are auto-generated from a lookup column) + if c.get("AttributeOf"): + continue + # Extract display label + label = "" + dn = c.get("DisplayName") + if isinstance(dn, dict): + ul = dn.get("UserLocalizedLabel") + if isinstance(ul, dict): + label = ul.get("Label", "") + result.append( + { + "name": name, + "type": c.get("AttributeType", ""), + "is_pk": bool(c.get("IsPrimaryId")), + "is_name": bool(c.get("IsPrimaryName")), + "label": label, + } + ) + result.sort(key=lambda x: (not x["is_pk"], not x["is_name"], x["name"])) + return result + + # ========================================================================= + # OData helpers -- discover columns, navigation properties, and bind values + # ========================================================================= + + # ------------------------------------------------------- odata_expands + + async def odata_expands( + self, + table: str, + ) -> List[Dict[str, Any]]: + """Discover all ``$expand`` navigation properties from a table. + + Returns entries for each outgoing lookup (single-valued navigation + property). Each entry contains the exact PascalCase navigation + property name needed for ``$expand`` and ``@odata.bind``, plus + the target entity set name. + + :param table: Schema name of the table (e.g. ``"contact"``). + :type table: :class:`str` + + :return: List of dicts, each with: + + - ``nav_property`` -- PascalCase navigation property for $expand + - ``target_table`` -- target entity logical name + - ``target_entity_set`` -- target entity set (for @odata.bind) + - ``lookup_attribute`` -- the lookup column logical name + - ``relationship`` -- relationship schema name + + :rtype: list[dict[str, typing.Any]] + + Example:: + + expands = await client.query.odata_expands("contact") + for e in expands: + print(f"expand={e['nav_property']} -> {e['target_table']}") + + # Use in a query + e = next(e for e in expands if e['target_table'] == 'account') + records = await client.records.list("contact", + select=["fullname"], + expand=[e['nav_property']]) + """ + table_lower = table.lower() + rels = await self._client.tables.list_table_relationships(table) + + result: List[Dict[str, Any]] = [] + for r in rels: + ref_entity = (r.get("ReferencingEntity") or "").lower() + if ref_entity != table_lower: + continue + nav_prop = r.get("ReferencingEntityNavigationPropertyName", "") + target = r.get("ReferencedEntity", "") + lookup_attr = r.get("ReferencingAttribute", "") + schema = r.get("SchemaName", "") + if not nav_prop or not target: + continue + + # Resolve entity set name for @odata.bind + target_set = "" + try: + async with self._client._scoped_odata() as od: + target_set = await od._entity_set_from_schema_name(target) + except (KeyError, AttributeError, ValueError, MetadataError): + pass # Entity set resolution failed; target_set stays empty + + result.append( + { + "nav_property": nav_prop, + "target_table": target, + "target_entity_set": target_set, + "lookup_attribute": lookup_attr, + "relationship": schema, + } + ) + + result.sort(key=lambda x: (x["target_table"], x["nav_property"])) + return result diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_records.py b/src/PowerPlatform/Dataverse/aio/operations/async_records.py new file mode 100644 index 00000000..1a2d86cf --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_records.py @@ -0,0 +1,522 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async record CRUD operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Any, AsyncGenerator, Dict, List, Optional, Union, overload, TYPE_CHECKING + +from ...core.errors import HttpError +from ...models.record import QueryResult, Record +from ...models.upsert import UpsertItem + +if TYPE_CHECKING: + from ...models.filters import FilterExpression + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncRecordOperations"] + + +class AsyncRecordOperations: + """Async namespace for record-level CRUD operations. + + Accessed via ``client.records``. Provides create, update, delete, retrieve, + list, and upsert operations on individual Dataverse records. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Create a single record + guid = await client.records.create("account", {"name": "Contoso Ltd"}) + + # Retrieve a record + record = await client.records.retrieve("account", guid, select=["name"]) + + # Update a record + await client.records.update("account", guid, {"telephone1": "555-0100"}) + + # Delete a record + await client.records.delete("account", guid) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ------------------------------------------------------------------ create + + @overload + async def create(self, table: str, data: Dict[str, Any]) -> str: ... + + @overload + async def create(self, table: str, data: List[Dict[str, Any]]) -> List[str]: ... + + async def create( + self, + table: str, + data: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> Union[str, List[str]]: + """Create one or more records in a Dataverse table. + + When ``data`` is a single dictionary, creates one record and returns its + GUID as a string. When ``data`` is a list of dictionaries, creates all + records via the ``CreateMultiple`` action and returns a list of GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param data: A single record dictionary or a list of record dictionaries. + Each dictionary maps column schema names to values. + :type data: dict or list[dict] + + :return: A single GUID string for a single record, or a list of GUID + strings for bulk creation. + :rtype: str or list[str] + + :raises TypeError: If ``data`` is not a dict or list[dict]. + + Example: + Create a single record:: + + guid = await client.records.create("account", {"name": "Contoso"}) + print(f"Created: {guid}") + + Create multiple records:: + + guids = await client.records.create("account", [ + {"name": "Contoso"}, + {"name": "Fabrikam"}, + ]) + print(f"Created {len(guids)} accounts") + """ + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(table) + if isinstance(data, dict): + rid = await od._create(entity_set, table, data) + if not isinstance(rid, str): + raise TypeError("_create (single) did not return GUID string") + return rid + if isinstance(data, list): + ids = await od._create_multiple(entity_set, table, data) + if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): + raise TypeError("_create (multi) did not return list[str]") + return ids + raise TypeError("data must be dict or list[dict]") + + # ------------------------------------------------------------------ update + + async def update( + self, + table: str, + ids: Union[str, List[str]], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """Update one or more records in a Dataverse table. + + Supports three usage patterns: + + 1. **Single** -- ``update("account", "guid", {"name": "New"})`` + 2. **Broadcast** -- ``update("account", [id1, id2], {"status": 1})`` + applies the same changes dict to every ID. + 3. **Paired** -- ``update("account", [id1, id2], [ch1, ch2])`` + applies each changes dict to its corresponding ID (lists must be + equal length). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param ids: A single GUID string, or a list of GUID strings. + :type ids: str or list[str] + :param changes: A dictionary of field changes (single/broadcast), or a + list of dictionaries (paired, one per ID). + :type changes: dict or list[dict] + + :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` + does not match the expected pattern. + + Example: + Single update:: + + await client.records.update("account", account_id, {"telephone1": "555-0100"}) + + Broadcast update:: + + await client.records.update("account", [id1, id2], {"statecode": 1}) + + Paired update:: + + await client.records.update( + "account", + [id1, id2], + [{"name": "Name A"}, {"name": "Name B"}], + ) + """ + async with self._client._scoped_odata() as od: + if isinstance(ids, str): + if not isinstance(changes, dict): + raise TypeError("For single id, changes must be a dict") + await od._update(table, ids, changes) + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + await od._update_by_ids(table, ids, changes) + return None + + # ------------------------------------------------------------------ delete + + @overload + async def delete(self, table: str, ids: str) -> None: ... + + @overload + async def delete(self, table: str, ids: List[str], *, use_bulk_delete: bool = True) -> Optional[str]: ... + + async def delete( + self, + table: str, + ids: Union[str, List[str]], + *, + use_bulk_delete: bool = True, + ) -> Optional[str]: + """Delete one or more records from a Dataverse table. + + When ``ids`` is a single string, deletes that one record. When ``ids`` + is a list, either executes a BulkDelete action (returning the async job + ID) or deletes each record sequentially depending on ``use_bulk_delete``. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param ids: A single GUID string, or a list of GUID strings. + :type ids: str or list[str] + :param use_bulk_delete: When True (default) and ``ids`` is a list, use + the BulkDelete action and return its async job ID. When False, delete + records one at a time. + :type use_bulk_delete: :class:`bool` + + :return: The BulkDelete job ID when bulk-deleting; otherwise None. + :rtype: :class:`str` or None + + :raises TypeError: If ``ids`` is not str or list[str]. + + Example: + Delete a single record:: + + await client.records.delete("account", account_id) + + Bulk delete:: + + job_id = await client.records.delete("account", [id1, id2, id3]) + """ + async with self._client._scoped_odata() as od: + if isinstance(ids, str): + await od._delete(table, ids) + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + if not ids: + return None + if not all(isinstance(rid, str) for rid in ids): + raise TypeError("ids must contain string GUIDs") + if use_bulk_delete: + return await od._delete_multiple(table, ids) + for rid in ids: + await od._delete(table, rid) + return None + + # --------------------------------------------------------------- retrieve + + async def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Optional[Record]: + """Fetch a single record by its GUID, returning ``None`` if not found. + + Returns ``None`` instead of raising when the record does not exist (HTTP 404). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand (e.g. + ``["primarycontactid"]``). Navigation property names are + case-sensitive and must match the entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + :return: Typed record, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None + + Example:: + + record = await client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + if record is not None: + contact = record.get("primarycontactid") or {} + print(contact.get("fullname")) + """ + async with self._client._scoped_odata() as od: + try: + raw = await od._get( + table, record_id, select=select, expand=expand, include_annotations=include_annotations + ) + except HttpError as exc: + if exc.status_code == 404: + return None + raise + return Record.from_api_response(table, raw, record_id=record_id) + + # -------------------------------------------------------------------- list + + async def list( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> QueryResult: + """Fetch multiple records and return them as a :class:`QueryResult`. + + All pages are collected eagerly and returned as a single :class:`QueryResult`. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: All matching records collected into a :class:`QueryResult`. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + from PowerPlatform.Dataverse import col + + result = await client.records.list( + "account", + filter=col("statecode") == 0, + select=["name", "statuscode"], + orderby=["name asc"], + top=100, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + for record in result: + print(record["name"], record.get("statuscode@OData.Community.Display.V1.FormattedValue")) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + all_records: List[Record] = [] + async with self._client._scoped_odata() as od: + async for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + all_records.extend(Record.from_api_response(table, row) for row in page) + return QueryResult(all_records) + + # --------------------------------------------------------------- list_pages + + async def list_pages( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> AsyncGenerator[QueryResult, None]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming counterpart to :meth:`list` — use when you want to process + records page by page without loading all into memory. Each iteration + triggers one network request via ``@odata.nextLink``. One-shot — do + not iterate more than once. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions. + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: Async generator of per-page :class:`QueryResult` objects. + :rtype: AsyncGenerator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`, None] + + Example:: + + async for page in client.records.list_pages( + "account", + filter="statecode eq 0", + orderby=["name asc"], + page_size=200, + ): + process(page.to_dataframe()) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + async with self._client._scoped_odata() as od: + async for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + yield QueryResult([Record.from_api_response(table, row) for row in page]) + + # ------------------------------------------------------------------ upsert + + async def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> None: + """Upsert one or more records identified by alternate keys. + + When ``items`` contains a single entry, performs a single upsert via PATCH + using the alternate key in the URL. When ``items`` contains multiple entries, + uses the ``UpsertMultiple`` bulk action. + + Each item must be either a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + or a plain ``dict`` with ``"alternate_key"`` and ``"record"`` keys (both dicts). + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: str + :param items: Non-empty list of :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + instances or dicts with ``"alternate_key"`` and ``"record"`` keys. + :type items: list[UpsertItem | dict] + + :return: ``None`` + :rtype: None + + :raises TypeError: If ``items`` is not a non-empty list, or if any element is + neither a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` nor a + dict with ``"alternate_key"`` and ``"record"`` keys. + + Example: + Upsert a single record using ``UpsertItem``:: + + from PowerPlatform.Dataverse.models.upsert import UpsertItem + + await client.records.upsert("account", [ + UpsertItem( + alternate_key={"accountnumber": "ACC-001"}, + record={"name": "Contoso Ltd", "description": "Primary account"}, + ) + ]) + + Upsert a single record using a plain dict:: + + await client.records.upsert("account", [ + { + "alternate_key": {"accountnumber": "ACC-001"}, + "record": {"name": "Contoso Ltd", "description": "Primary account"}, + }, + ]) + + Upsert multiple records using ``UpsertItem``:: + + from PowerPlatform.Dataverse.models.upsert import UpsertItem + + await client.records.upsert("account", [ + UpsertItem( + alternate_key={"accountnumber": "ACC-001"}, + record={"name": "Contoso Ltd", "description": "Primary account"}, + ), + UpsertItem( + alternate_key={"accountnumber": "ACC-002"}, + record={"name": "Fabrikam Inc", "description": "Partner account"}, + ), + ]) + + Upsert multiple records using plain dicts:: + + await client.records.upsert("account", [ + { + "alternate_key": {"accountnumber": "ACC-001"}, + "record": {"name": "Contoso Ltd", "description": "Primary account"}, + }, + { + "alternate_key": {"accountnumber": "ACC-002"}, + "record": {"name": "Fabrikam Inc", "description": "Partner account"}, + }, + ]) + + The ``alternate_key`` dict may contain multiple columns when the configured + alternate key is composite, e.g. + ``{"accountnumber": "ACC-001", "address1_postalcode": "98052"}``. + """ + if not isinstance(items, list) or not items: + raise TypeError("items must be a non-empty list of UpsertItem or dicts") + normalized: List[UpsertItem] = [] + for i in items: + if isinstance(i, UpsertItem): + normalized.append(i) + elif isinstance(i, dict) and isinstance(i.get("alternate_key"), dict) and isinstance(i.get("record"), dict): + normalized.append(UpsertItem(alternate_key=i["alternate_key"], record=i["record"])) + else: + raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(table) + if len(normalized) == 1: + item = normalized[0] + await od._upsert(entity_set, table, item.alternate_key, item.record) + else: + alternate_keys = [i.alternate_key for i in normalized] + records = [i.record for i in normalized] + await od._upsert_multiple(entity_set, table, alternate_keys, records) + return None diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_tables.py b/src/PowerPlatform/Dataverse/aio/operations/async_tables.py new file mode 100644 index 00000000..0fbe61c8 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_tables.py @@ -0,0 +1,838 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async table metadata operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING + +from ...models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + RelationshipInfo, +) +from ...models.table_info import AlternateKeyInfo +from ...models.labels import Label, LocalizedLabel +from ...models.table_info import TableInfo +from ...common.constants import CASCADE_BEHAVIOR_REMOVE_LINK + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncTableOperations"] + + +class AsyncTableOperations: + """Async namespace for table-level metadata operations. + + Accessed via ``client.tables``. Provides operations to create, delete, + inspect, and list Dataverse tables, as well as add and remove columns. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Create a table + info = await client.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", + ) + + # List tables + tables = await client.tables.list() + + # Get table info + info = await client.tables.get("new_Product") + + # Add columns + await client.tables.add_columns("new_Product", {"new_Rating": "int"}) + + # Remove columns + await client.tables.remove_columns("new_Product", "new_Rating") + + # Delete a table + await client.tables.delete("new_Product") + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- create + + async def create( + self, + table: str, + columns: Dict[str, Any], + *, + solution: Optional[str] = None, + primary_column: Optional[str] = None, + display_name: Optional[str] = None, + ) -> TableInfo: + """Create a custom table with the specified columns. + + :param table: Schema name of the table with customization prefix + (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Mapping of column schema names (with customization + prefix) to their types. Supported types include ``"string"`` + (or ``"text"``), ``"memo"`` (or ``"multiline"``), + ``"int"`` (or ``"integer"``), ``"decimal"`` + (or ``"money"``), ``"float"`` (or ``"double"``), ``"datetime"`` + (or ``"date"``), ``"bool"`` (or ``"boolean"``), ``"file"``, and + ``Enum`` subclasses + (for local option sets). + :type columns: :class:`dict` + :param solution: Optional solution unique name that should own the new + table. When omitted the table is created in the default solution. + :type solution: :class:`str` or None + :param primary_column: Optional primary name column schema name with + customization prefix (e.g. ``"new_ProductName"``). If not provided, + defaults to ``"{prefix}_Name"``. + :type primary_column: :class:`str` or None + :param display_name: Human-readable display name for the table + (e.g. ``"Product"``). When omitted, defaults to the table schema name. + :type display_name: :class:`str` or None + + :return: Table metadata with ``schema_name``, ``entity_set_name``, + ``logical_name``, ``metadata_id``, and ``columns_created``. + Supports dict-like access with legacy keys for backward + compatibility. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If table creation fails or the table already exists. + + Example: + Create a table with simple columns:: + + from enum import IntEnum + + class ItemStatus(IntEnum): + ACTIVE = 1 + INACTIVE = 2 + + result = await client.tables.create( + "new_Product", + { + "new_Title": "string", + "new_Price": "decimal", + "new_Status": ItemStatus, + }, + solution="MySolution", + primary_column="new_ProductName", + display_name="Product", + ) + print(f"Created: {result['table_schema_name']}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_table( + table, + columns, + solution, + primary_column, + display_name, + ) + return TableInfo.from_dict(raw) + + # ----------------------------------------------------------------- delete + + async def delete(self, table: str) -> None: + """Delete a custom table by schema name. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist or deletion fails. + + .. warning:: + This operation is irreversible and will delete all records in the + table along with the table definition. + + Example:: + + await client.tables.delete("new_MyTestTable") + """ + async with self._client._scoped_odata() as od: + await od._delete_table(table) + + # -------------------------------------------------------------------- get + + async def get(self, table: str) -> Optional[TableInfo]: + """Get basic metadata for a table if it exists. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"`` + or ``"account"``). + :type table: :class:`str` + + :return: Table metadata, or ``None`` if the table is not found. + Supports dict-like access with legacy keys for backward + compatibility. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + or None + + Example:: + + info = await client.tables.get("new_MyTestTable") + if info: + print(f"Logical name: {info['table_logical_name']}") + print(f"Entity set: {info['entity_set_name']}") + """ + async with self._client._scoped_odata() as od: + raw = await od._get_table_info(table) + if raw is None: + return None + return TableInfo.from_dict(raw) + + # ------------------------------------------------------------------- list + + async def list( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all non-private tables in the Dataverse environment. + + By default returns every table where ``IsPrivate eq false``. Supply + an optional OData ``$filter`` expression to further narrow the results. + The expression is combined with the default ``IsPrivate eq false`` + clause using ``and``. + + :param filter: Optional OData ``$filter`` expression to further narrow + the list of returned tables (e.g. + ``"SchemaName eq 'Account'"``). Column names in filter + expressions must use the exact property names from the + ``EntityDefinitions`` metadata (typically PascalCase). + :type filter: :class:`str` or None + :param select: Optional list of property names to include in the + response (projected via the OData ``$select`` query option). + Property names must use the exact PascalCase names from the + ``EntityDefinitions`` metadata (e.g. + ``["LogicalName", "SchemaName", "DisplayName"]``). + When ``None`` (the default) or an empty list, all properties are + returned. + :type select: list[str] or None + + :return: List of EntityDefinition metadata dictionaries. + :rtype: list[dict] + + Example:: + + # List all non-private tables + tables = await client.tables.list() + for table in tables: + print(table["LogicalName"]) + + # List only tables whose schema name starts with "new_" + custom_tables = await client.tables.list( + filter="startswith(SchemaName, 'new_')" + ) + + # List tables with only specific properties + tables = await client.tables.list( + select=["LogicalName", "SchemaName", "EntitySetName"] + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_tables(filter=filter, select=select) + + # ------------------------------------------------------------- add_columns + + async def add_columns( + self, + table: str, + columns: Dict[str, Any], + ) -> List[str]: + """Add one or more columns to an existing table. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Mapping of column schema names (with customization + prefix) to their types. Supported types are the same as for + :meth:`create`. + :type columns: :class:`dict` + + :return: Schema names of the columns that were created. + :rtype: list[str] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + + Example:: + + created = await client.tables.add_columns( + "new_MyTestTable", + {"new_Notes": "string", "new_Active": "bool"}, + ) + print(created) # ['new_Notes', 'new_Active'] + """ + async with self._client._scoped_odata() as od: + return await od._create_columns(table, columns) + + # ---------------------------------------------------------- remove_columns + + async def remove_columns( + self, + table: str, + columns: Union[str, List[str]], + ) -> List[str]: + """Remove one or more columns from a table. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Column schema name or list of column schema names to + remove. Must include the customization prefix (e.g. + ``"new_TestColumn"``). + :type columns: str or list[str] + + :return: Schema names of the columns that were removed. + :rtype: list[str] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table or a specified column does not exist. + + Example:: + + removed = await client.tables.remove_columns( + "new_MyTestTable", + ["new_Notes", "new_Active"], + ) + print(removed) # ['new_Notes', 'new_Active'] + """ + async with self._client._scoped_odata() as od: + return await od._delete_columns(table, columns) + + # ------------------------------------------------------ create_one_to_many + + async def create_one_to_many_relationship( + self, + lookup: LookupAttributeMetadata, + relationship: OneToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> RelationshipInfo: + """Create a one-to-many relationship between tables. + + This operation creates both the relationship and the lookup attribute + on the referencing table. + + :param lookup: Metadata defining the lookup attribute. + :type lookup: ~PowerPlatform.Dataverse.models.relationship.LookupAttributeMetadata + :param relationship: Metadata defining the relationship. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.OneToManyRelationshipMetadata + :param solution: Optional solution unique name to add relationship to. + :type solution: :class:`str` or None + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``lookup_schema_name``, ``referenced_entity``, and + ``referencing_entity``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a one-to-many relationship: Department (1) -> Employee (N):: + + from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + Label, + LocalizedLabel, + CascadeConfiguration, + ) + from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_REMOVE_LINK, + ) + + lookup = LookupAttributeMetadata( + schema_name="new_DepartmentId", + display_name=Label( + localized_labels=[ + LocalizedLabel(label="Department", language_code=1033) + ] + ), + ) + + relationship = OneToManyRelationshipMetadata( + schema_name="new_Department_Employee", + referenced_entity="new_department", + referencing_entity="new_employee", + referenced_attribute="new_departmentid", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ), + ) + + result = await client.tables.create_one_to_many_relationship(lookup, relationship) + print(f"Created lookup field: {result.lookup_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_one_to_many_relationship( + lookup, + relationship, + solution, + ) + return RelationshipInfo.from_one_to_many( + relationship_id=raw["relationship_id"], + relationship_schema_name=raw["relationship_schema_name"], + lookup_schema_name=raw["lookup_schema_name"], + referenced_entity=raw["referenced_entity"], + referencing_entity=raw["referencing_entity"], + ) + + # ----------------------------------------------------- create_many_to_many + + async def create_many_to_many_relationship( + self, + relationship: ManyToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> RelationshipInfo: + """Create a many-to-many relationship between tables. + + This operation creates a many-to-many relationship and an intersect + table to manage the relationship. + + :param relationship: Metadata defining the many-to-many relationship. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.ManyToManyRelationshipMetadata + :param solution: Optional solution unique name to add relationship to. + :type solution: :class:`str` or None + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``entity1_logical_name``, and ``entity2_logical_name``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a many-to-many relationship: Employee <-> Project:: + + from PowerPlatform.Dataverse.models.relationship import ( + ManyToManyRelationshipMetadata, + ) + + relationship = ManyToManyRelationshipMetadata( + schema_name="new_employee_project", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + + result = await client.tables.create_many_to_many_relationship(relationship) + print(f"Created: {result.relationship_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_many_to_many_relationship( + relationship, + solution, + ) + return RelationshipInfo.from_many_to_many( + relationship_id=raw["relationship_id"], + relationship_schema_name=raw["relationship_schema_name"], + entity1_logical_name=raw["entity1_logical_name"], + entity2_logical_name=raw["entity2_logical_name"], + ) + + # ------------------------------------------------------- delete_relationship + + async def delete_relationship(self, relationship_id: str) -> None: + """Delete a relationship by its metadata ID. + + :param relationship_id: The GUID of the relationship metadata. + :type relationship_id: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + .. warning:: + Deleting a relationship also removes the associated lookup attribute + for one-to-many relationships. This operation is irreversible. + + Example:: + + await client.tables.delete_relationship( + "12345678-1234-1234-1234-123456789abc" + ) + """ + async with self._client._scoped_odata() as od: + await od._delete_relationship(relationship_id) + + # -------------------------------------------------------- get_relationship + + async def get_relationship(self, schema_name: str) -> Optional[RelationshipInfo]: + """Retrieve relationship metadata by schema name. + + :param schema_name: The schema name of the relationship. + :type schema_name: :class:`str` + + :return: Relationship metadata, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + or None + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + rel = await client.tables.get_relationship("new_Department_Employee") + if rel: + print(f"Found: {rel.relationship_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._get_relationship(schema_name) + if raw is None: + return None + return RelationshipInfo.from_api_response(raw) + + # ------------------------------------------------------- create_lookup_field + + async def create_lookup_field( + self, + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + solution: Optional[str] = None, + language_code: int = 1033, + ) -> RelationshipInfo: + """Create a simple lookup field relationship. + + This is a convenience method that wraps :meth:`create_one_to_many_relationship` + for the common case of adding a lookup field to an existing table. + + :param referencing_table: Logical name of the table that will have + the lookup field (child table). + :type referencing_table: :class:`str` + :param lookup_field_name: Schema name for the lookup field + (e.g., ``"new_AccountId"``). + :type lookup_field_name: :class:`str` + :param referenced_table: Logical name of the table being referenced + (parent table). + :type referenced_table: :class:`str` + :param display_name: Display name for the lookup field. Defaults to + the referenced table name. + :type display_name: :class:`str` or None + :param description: Optional description for the lookup field. + :type description: :class:`str` or None + :param required: Whether the lookup is required. Defaults to ``False``. + :type required: :class:`bool` + :param cascade_delete: Delete behavior (``"RemoveLink"``, + ``"Cascade"``, ``"Restrict"``). Defaults to ``"RemoveLink"``. + :type cascade_delete: :class:`str` + :param solution: Optional solution unique name to add the relationship + to. + :type solution: :class:`str` or None + :param language_code: Language code for labels. Defaults to 1033 + (English). + :type language_code: :class:`int` + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``lookup_schema_name``, ``referenced_entity``, and + ``referencing_entity``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a simple lookup field:: + + result = await client.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + display_name="Account", + required=True, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + print(f"Created lookup: {result['lookup_schema_name']}") + """ + async with self._client._scoped_odata() as od: + lookup, relationship = od._build_lookup_field_models( + referencing_table=referencing_table, + lookup_field_name=lookup_field_name, + referenced_table=referenced_table, + display_name=display_name, + description=description, + required=required, + cascade_delete=cascade_delete, + language_code=language_code, + ) + + return await self.create_one_to_many_relationship(lookup, relationship, solution=solution) + + # ------------------------------------------------- create_alternate_key + + async def create_alternate_key( + self, + table: str, + key_name: str, + columns: List[str], + *, + display_name: Optional[str] = None, + language_code: int = 1033, + ) -> AlternateKeyInfo: + """Create an alternate key on a table. + + Alternate keys allow upsert operations to identify records by one or + more columns instead of the primary GUID. After creation the key is + queued for index building; its :attr:`~AlternateKeyInfo.status` will + transition from ``"Pending"`` to ``"Active"`` once the index is ready. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + :param key_name: Schema name for the new alternate key + (e.g. ``"new_product_code_key"``). + :type key_name: :class:`str` + :param columns: List of column logical names that compose the key + (e.g. ``["new_productcode"]``). + :type columns: list[str] + :param display_name: Display name for the key. Defaults to + ``key_name`` if not provided. + :type display_name: :class:`str` or None + :param language_code: Language code for labels. Defaults to 1033 + (English). + :type language_code: :class:`int` + + :return: Metadata for the newly created alternate key. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a single-column alternate key for upsert:: + + key = await client.tables.create_alternate_key( + "new_Product", + "new_product_code_key", + ["new_productcode"], + display_name="Product Code", + ) + print(f"Key ID: {key.metadata_id}") + print(f"Columns: {key.key_attributes}") + """ + label = Label(localized_labels=[LocalizedLabel(label=display_name or key_name, language_code=language_code)]) + async with self._client._scoped_odata() as od: + raw = await od._create_alternate_key(table, key_name, columns, label) + return AlternateKeyInfo( + metadata_id=raw["metadata_id"], + schema_name=raw["schema_name"], + key_attributes=raw["key_attributes"], + status="Pending", + ) + + # --------------------------------------------------- get_alternate_keys + + async def get_alternate_keys(self, table: str) -> List[AlternateKeyInfo]: + """List all alternate keys defined on a table. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + + :return: List of alternate key metadata objects. May be empty if no + alternate keys are defined. + :rtype: list[~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + List alternate keys and print their status:: + + keys = await client.tables.get_alternate_keys("new_Product") + for key in keys: + print(f"{key.schema_name}: {key.status}") + """ + async with self._client._scoped_odata() as od: + raw_list = await od._get_alternate_keys(table) + return [AlternateKeyInfo.from_api_response(item) for item in raw_list] + + # ------------------------------------------------ delete_alternate_key + + async def delete_alternate_key(self, table: str, key_id: str) -> None: + """Delete an alternate key by its metadata ID. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + :param key_id: Metadata GUID of the alternate key to delete. + :type key_id: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + .. warning:: + Deleting an alternate key that is in use by upsert operations will + cause those operations to fail. This operation is irreversible. + + Example:: + + await client.tables.delete_alternate_key( + "new_Product", + "12345678-1234-1234-1234-123456789abc", + ) + """ + async with self._client._scoped_odata() as od: + await od._delete_alternate_key(table, key_id) + + # -------------------------------------------------------- list_columns + + async def list_columns( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """List all attribute (column) definitions for a table. + + :param table: Schema name of the table (e.g. ``"account"`` or + ``"new_Product"``). + :type table: :class:`str` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + :param filter: Optional OData ``$filter`` expression. For example, + ``"AttributeType eq 'String'"`` returns only string columns. + :type filter: :class:`str` or None + + :return: List of raw attribute metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table is not found. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all columns on the account table + columns = await client.tables.list_columns("account") + for col in columns: + print(f"{col['LogicalName']} ({col.get('AttributeType')})") + + # List only specific properties + columns = await client.tables.list_columns( + "account", + select=["LogicalName", "SchemaName", "AttributeType"], + ) + + # Filter to only string attributes + columns = await client.tables.list_columns( + "account", + filter="AttributeType eq 'String'", + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_columns(table, select=select, filter=filter) + + # ------------------------------------------------- list_relationships + + async def list_relationships( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationship definitions in the environment. + + :param filter: Optional OData ``$filter`` expression. For example, + ``"RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'"`` + returns only one-to-many relationships. + :type filter: :class:`str` or None + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + + :return: List of raw relationship metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all relationships + rels = await client.tables.list_relationships() + for rel in rels: + print(f"{rel['SchemaName']} ({rel.get('@odata.type')})") + + # Filter by type + one_to_many = await client.tables.list_relationships( + filter="RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'" + ) + + # Select specific properties + rels = await client.tables.list_relationships( + select=["SchemaName", "ReferencedEntity", "ReferencingEntity"] + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_relationships(filter=filter, select=select) + + # --------------------------------------------- list_table_relationships + + async def list_table_relationships( + self, + table: str, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationships for a specific table. + + Combines one-to-many, many-to-one, and many-to-many relationships + for the given table by querying + ``EntityDefinitions({id})/OneToManyRelationships``, + ``EntityDefinitions({id})/ManyToOneRelationships``, and + ``EntityDefinitions({id})/ManyToManyRelationships``. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData ``$filter`` expression applied to each + sub-request. + :type filter: :class:`str` or None + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + + :return: Combined list of one-to-many, many-to-one, and many-to-many + relationship metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table is not found. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all relationships for the account table + rels = await client.tables.list_table_relationships("account") + for rel in rels: + print(f"{rel['SchemaName']} -> {rel.get('@odata.type')}") + """ + async with self._client._scoped_odata() as od: + return await od._list_table_relationships(table, filter=filter, select=select) diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 12ceaaac..c9a1364f 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -105,7 +105,7 @@ def __init__( ) -> None: if config is not None and context is not None: raise ValueError( - "Cannot specify both 'config' and 'context'. Pass operation_context via DataverseConfig instead." + "Cannot specify both 'config' and 'context'. " "Pass operation_context via DataverseConfig instead." ) self.auth = _AuthManager(credential) self._base_url = (base_url or "").rstrip("/") diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index b848c5a0..8e2e85d9 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -5,239 +5,56 @@ from __future__ import annotations -import json -import re import uuid -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union - -from ..core.errors import HttpError, MetadataError, ValidationError -from ..core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND, _http_subcode -from ..models.batch import BatchItemResponse, BatchResult -from ..models.relationship import ( - LookupAttributeMetadata, - OneToManyRelationshipMetadata, - ManyToManyRelationshipMetadata, -) -from ..models.upsert import UpsertItem -from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from ..core.errors import MetadataError, ValidationError +from ..core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND +from ..models.batch import BatchResult from ._raw_request import _RawRequest -from ._odata import _GUID_RE +from ._batch_base import ( + _BatchBase, + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, + _ChangeSetBatchItem, + _MAX_BATCH_SIZE, + _CRLF, + _extract_boundary, + _raise_top_level_batch_error, + _split_multipart, + _parse_mime_part, + _parse_http_response_part, +) if TYPE_CHECKING: from ._odata import _ODataClient __all__ = [] -_CRLF = "\r\n" -_MAX_BATCH_SIZE = 1000 - - -# --------------------------------------------------------------------------- -# Intent dataclasses — one per supported operation type -# (stored at batch-build time; resolved to _RawRequest at execute() time) -# --------------------------------------------------------------------------- - -# --- Record intent types --- - - -@dataclass -class _RecordCreate: - table: str - data: Union[Dict[str, Any], List[Dict[str, Any]]] - content_id: Optional[int] = None # set only for changeset items - - -@dataclass -class _RecordUpdate: - table: str - ids: Union[str, List[str]] - changes: Union[Dict[str, Any], List[Dict[str, Any]]] - content_id: Optional[int] = None # set only for changeset single-record updates - - -@dataclass -class _RecordDelete: - table: str - ids: Union[str, List[str]] - use_bulk_delete: bool = True - content_id: Optional[int] = None # set only for changeset single-record deletes - - -@dataclass -class _RecordGet: - table: str - record_id: str - select: Optional[List[str]] = None - expand: Optional[List[str]] = None - include_annotations: Optional[str] = None - - -@dataclass -class _RecordList: - table: str - select: Optional[List[str]] = None - filter: Optional[str] = None - orderby: Optional[List[str]] = None - top: Optional[int] = None - expand: Optional[List[str]] = None - page_size: Optional[int] = None - count: bool = False - include_annotations: Optional[str] = None - - -@dataclass -class _RecordUpsert: - table: str - items: List[UpsertItem] # always non-empty; normalised by BatchRecordOperations - - -# --- Table intent types --- - - -@dataclass -class _TableCreate: - table: str - columns: Dict[str, Any] - solution: Optional[str] = None - primary_column: Optional[str] = None - display_name: Optional[str] = None - - -@dataclass -class _TableDelete: - table: str - - -@dataclass -class _TableGet: - table: str - - -@dataclass -class _TableList: - filter: Optional[str] = None - select: Optional[List[str]] = None - - -@dataclass -class _TableAddColumns: - table: str - columns: Dict[str, Any] - - -@dataclass -class _TableRemoveColumns: - table: str - columns: Union[str, List[str]] - - -@dataclass -class _TableCreateOneToMany: - lookup: LookupAttributeMetadata - relationship: OneToManyRelationshipMetadata - solution: Optional[str] = None - - -@dataclass -class _TableCreateManyToMany: - relationship: ManyToManyRelationshipMetadata - solution: Optional[str] = None - - -@dataclass -class _TableDeleteRelationship: - relationship_id: str - - -@dataclass -class _TableGetRelationship: - schema_name: str - - -@dataclass -class _TableCreateLookupField: - referencing_table: str - lookup_field_name: str - referenced_table: str - display_name: Optional[str] = None - description: Optional[str] = None - required: bool = False - cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK - solution: Optional[str] = None - language_code: int = 1033 - - -# --- Query intent types --- - - -@dataclass -class _QuerySql: - sql: str - - -# --------------------------------------------------------------------------- -# Changeset container -# --------------------------------------------------------------------------- - - -@dataclass -class _ChangeSet: - """Ordered group of single-record write operations that execute atomically. - - Content-IDs are allocated from ``_counter``, a single-element ``List[int]`` - that is shared across all changesets in the same batch. Passing the same - list object to every ``_ChangeSet`` created by a :class:`BatchRequest` - ensures Content-ID values are unique within the entire batch request, not - just within an individual changeset, as required by the OData spec. - - When constructed in isolation (e.g. in unit tests), ``_counter`` defaults - to a fresh ``[1]`` so the class remains self-contained. - """ - - operations: List[Union[_RecordCreate, _RecordUpdate, _RecordDelete]] = field(default_factory=list) - _counter: List[int] = field(default_factory=lambda: [1], repr=False) - - def add_create(self, table: str, data: Dict[str, Any]) -> str: - """Add a single-record create; return its content-ID reference string.""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordCreate(table=table, data=data, content_id=cid)) - return f"${cid}" - - def add_update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: - """Add a single-record update (record_id may be a '$n' reference).""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordUpdate(table=table, ids=record_id, changes=changes, content_id=cid)) - - def add_delete(self, table: str, record_id: str) -> None: - """Add a single-record delete (record_id may be a '$n' reference).""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordDelete(table=table, ids=record_id, content_id=cid)) - - -# --------------------------------------------------------------------------- -# Changeset batch item -# (_RawRequest is imported from ._raw_request — defined there so _odata.py -# can also import it without a circular dependency) -# --------------------------------------------------------------------------- - - -@dataclass -class _ChangeSetBatchItem: - """A resolved changeset — serialised as a nested multipart/mixed part.""" - - requests: List[_RawRequest] - # --------------------------------------------------------------------------- # Batch client: resolves intents → raw requests → multipart body → HTTP → result # --------------------------------------------------------------------------- -class _BatchClient: +class _BatchClient(_BatchBase): """ Serialises a list of intent objects into an OData ``$batch`` multipart/mixed request, dispatches it, and parses the response. @@ -245,9 +62,6 @@ class _BatchClient: :param od: The active OData client (provides helpers and HTTP transport). """ - def __init__(self, od: "_ODataClient") -> None: - self._od = od - # ------------------------------------------------------------------ # Public entry point # ------------------------------------------------------------------ @@ -449,19 +263,10 @@ def _require_entity_metadata(self, table: str) -> str: ) return ent["MetadataId"] - def _resolve_table_create(self, op: _TableCreate) -> List[_RawRequest]: - return [self._od._build_create_entity(op.table, op.columns, op.solution, op.primary_column, op.display_name)] - def _resolve_table_delete(self, op: _TableDelete) -> List[_RawRequest]: metadata_id = self._require_entity_metadata(op.table) return [self._od._build_delete_entity(metadata_id)] - def _resolve_table_get(self, op: _TableGet) -> List[_RawRequest]: - return [self._od._build_get_entity(op.table)] - - def _resolve_table_list(self, op: _TableList) -> List[_RawRequest]: - return [self._od._build_list_entities(filter=op.filter, select=op.select)] - def _resolve_table_add_columns(self, op: _TableAddColumns) -> List[_RawRequest]: metadata_id = self._require_entity_metadata(op.table) return [self._od._build_create_column(metadata_id, col_name, dtype) for col_name, dtype in op.columns.items()] @@ -482,255 +287,9 @@ def _resolve_table_remove_columns(self, op: _TableRemoveColumns) -> List[_RawReq requests.append(self._od._build_delete_column(metadata_id, attr_meta["MetadataId"])) return requests - def _resolve_table_create_one_to_many(self, op: _TableCreateOneToMany) -> List[_RawRequest]: - body = op.relationship.to_dict() - body["Lookup"] = op.lookup.to_dict() - return [self._od._build_create_relationship(body, solution=op.solution)] - - def _resolve_table_create_many_to_many(self, op: _TableCreateManyToMany) -> List[_RawRequest]: - return [self._od._build_create_relationship(op.relationship.to_dict(), solution=op.solution)] - - def _resolve_table_delete_relationship(self, op: _TableDeleteRelationship) -> List[_RawRequest]: - return [self._od._build_delete_relationship(op.relationship_id)] - - def _resolve_table_get_relationship(self, op: _TableGetRelationship) -> List[_RawRequest]: - return [self._od._build_get_relationship(op.schema_name)] - - def _resolve_table_create_lookup_field(self, op: _TableCreateLookupField) -> List[_RawRequest]: - lookup, relationship = self._od._build_lookup_field_models( - referencing_table=op.referencing_table, - lookup_field_name=op.lookup_field_name, - referenced_table=op.referenced_table, - display_name=op.display_name, - description=op.description, - required=op.required, - cascade_delete=op.cascade_delete, - language_code=op.language_code, - ) - body = relationship.to_dict() - body["Lookup"] = lookup.to_dict() - return [self._od._build_create_relationship(body, solution=op.solution)] - # ------------------------------------------------------------------ # Query resolvers — delegate to _ODataClient._build_* methods # ------------------------------------------------------------------ def _resolve_query_sql(self, op: _QuerySql) -> List[_RawRequest]: return [self._od._build_sql(op.sql)] - - # ------------------------------------------------------------------ - # Multipart serialisation - # ------------------------------------------------------------------ - - def _build_batch_body( - self, - resolved: List[Union[_RawRequest, _ChangeSetBatchItem]], - batch_boundary: str, - ) -> str: - parts: List[str] = [] - for item in resolved: - if isinstance(item, _ChangeSetBatchItem): - parts.append(self._serialize_changeset_item(item, batch_boundary)) - else: - parts.append(self._serialize_raw_request(item, batch_boundary)) - return "".join(parts) + f"--{batch_boundary}--{_CRLF}" - - def _serialize_raw_request(self, req: _RawRequest, boundary: str) -> str: - """Serialise a single operation as a multipart/mixed part with CRLF line endings.""" - part_header_lines = [ - f"--{boundary}", - "Content-Type: application/http", - "Content-Transfer-Encoding: binary", - ] - if req.content_id is not None: - part_header_lines.append(f"Content-ID: {req.content_id}") - - inner_lines = [f"{req.method} {req.url} HTTP/1.1"] - if req.body is not None: - inner_lines.append("Content-Type: application/json; type=entry") - if req.headers: - for k, v in req.headers.items(): - inner_lines.append(f"{k}: {v}") - inner_lines.append("") # blank line — end of inner headers - if req.body is not None: - inner_lines.append(req.body) - - part_header_str = _CRLF.join(part_header_lines) + _CRLF - inner_str = _CRLF.join(inner_lines) - return part_header_str + _CRLF + inner_str + _CRLF - - def _serialize_changeset_item(self, cs: _ChangeSetBatchItem, batch_boundary: str) -> str: - cs_boundary = f"changeset_{uuid.uuid4()}" - cs_parts = [self._serialize_raw_request(r, cs_boundary) for r in cs.requests] - cs_parts.append(f"--{cs_boundary}--{_CRLF}") - cs_body = "".join(cs_parts) - - outer = ( - f"--{batch_boundary}{_CRLF}" f'Content-Type: multipart/mixed; boundary="{cs_boundary}"{_CRLF}' f"{_CRLF}" - ) - return outer + cs_body + _CRLF - - # ------------------------------------------------------------------ - # Response parsing (multipart/mixed) - # ------------------------------------------------------------------ - - def _parse_batch_response(self, response: Any) -> BatchResult: - content_type = response.headers.get("Content-Type", "") - boundary = _extract_boundary(content_type) - if not boundary: - # Non-multipart response: the batch request itself was rejected by Dataverse - # (common for top-level 4xx, e.g. malformed body, missing OData headers). - # Returning an empty BatchResult() here would silently hide the error and - # make has_errors=False, which is actively misleading. Raise instead. - _raise_top_level_batch_error(response) - return BatchResult() # unreachable; satisfies type checkers - parts = _split_multipart(response.text or "", boundary) - responses: List[BatchItemResponse] = [] - for part_headers, part_body in parts: - part_ct = part_headers.get("content-type", "") - if "multipart/mixed" in part_ct: - inner_boundary = _extract_boundary(part_ct) - if inner_boundary: - for ih, ib in _split_multipart(part_body, inner_boundary): - item = _parse_http_response_part(ib, ih.get("content-id")) - if item is not None: - responses.append(item) - else: - item = _parse_http_response_part(part_body, content_id=part_headers.get("content-id")) - if item is not None: - responses.append(item) - return BatchResult(responses=responses) - - -# --------------------------------------------------------------------------- -# Multipart parsing helpers -# --------------------------------------------------------------------------- - - -def _raise_top_level_batch_error(response: Any) -> None: - """Parse a non-multipart batch response and raise HttpError with the service message. - - Dataverse returns ``application/json`` with an ``{"error": {...}}`` payload when - it rejects the batch request at the HTTP level (e.g. malformed multipart body, - missing OData headers). This helper surfaces that detail instead of silently - returning an empty ``BatchResult``. - """ - status_code: int = getattr(response, "status_code", 0) - service_error_code: Optional[str] = None - try: - payload = response.json() - error = payload.get("error", {}) - service_error_code = error.get("code") or None - message: str = error.get("message") or response.text or "Unexpected non-multipart response from $batch" - except Exception: - message = (getattr(response, "text", None) or "") or "Unexpected non-multipart response from $batch" - raise HttpError( - message=f"Batch request rejected by Dataverse: {message}", - status_code=status_code, - subcode=_http_subcode(status_code) if status_code else None, - service_error_code=service_error_code, - ) - - -_BOUNDARY_RE = re.compile(r'boundary="?([^";,\s]+)"?', re.IGNORECASE) - - -def _extract_boundary(content_type: str) -> Optional[str]: - m = _BOUNDARY_RE.search(content_type) - return m.group(1) if m else None - - -def _split_multipart(body: str, boundary: str) -> List[Tuple[Dict[str, str], str]]: - delimiter = f"--{boundary}" - parts: List[Tuple[Dict[str, str], str]] = [] - lines = body.replace("\r\n", "\n").split("\n") - current: List[str] = [] - in_part = False - for line in lines: - stripped = line.rstrip("\r") - if stripped == delimiter: - if in_part and current: - parts.append(_parse_mime_part("\n".join(current))) - current = [] - in_part = True - elif stripped == f"{delimiter}--": - if in_part and current: - parts.append(_parse_mime_part("\n".join(current))) - break - elif in_part: - current.append(line) - return parts - - -def _parse_mime_part(raw: str) -> Tuple[Dict[str, str], str]: - if "\n\n" in raw: - header_block, body = raw.split("\n\n", 1) - else: - header_block, body = raw, "" - headers: Dict[str, str] = {} - for line in header_block.splitlines(): - if ":" in line: - name, _, value = line.partition(":") - headers[name.strip().lower()] = value.strip() - return headers, body.strip() - - -def _parse_http_response_part(text: str, content_id: Optional[str]) -> Optional[BatchItemResponse]: - lines = text.replace("\r\n", "\n").splitlines() - if not lines: - return None - status_line = "" - idx = 0 - for i, line in enumerate(lines): - if line.startswith("HTTP/"): - status_line = line - idx = i + 1 - break - if not status_line: - return None - parts = status_line.split(" ", 2) - if len(parts) < 2: - return None - try: - status_code = int(parts[1]) - except ValueError: - return None - resp_headers: Dict[str, str] = {} - body_start = idx - for i in range(idx, len(lines)): - if lines[i] == "": - body_start = i + 1 - break - if ":" in lines[i]: - name, _, value = lines[i].partition(":") - resp_headers[name.strip().lower()] = value.strip() - entity_id: Optional[str] = None - odata_id = resp_headers.get("odata-entityid", "") - if odata_id: - m = _GUID_RE.search(odata_id) - if m: - entity_id = m.group(0) - body_text = "\n".join(lines[body_start:]).strip() - data: Optional[Dict[str, Any]] = None - error_message: Optional[str] = None - error_code: Optional[str] = None - if body_text: - try: - parsed = json.loads(body_text) - if isinstance(parsed, dict): - err = parsed.get("error") - if isinstance(err, dict): - error_message = err.get("message") - error_code = err.get("code") - else: - data = parsed - except (json.JSONDecodeError, ValueError): - pass - return BatchItemResponse( - status_code=status_code, - content_id=content_id, - entity_id=entity_id, - data=data, - error_message=error_message, - error_code=error_code, - ) diff --git a/src/PowerPlatform/Dataverse/data/_batch_base.py b/src/PowerPlatform/Dataverse/data/_batch_base.py new file mode 100644 index 00000000..46b247de --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_batch_base.py @@ -0,0 +1,513 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared intent types, multipart helpers, and pure-logic base for the Dataverse batch client. + +Contains no I/O. Subclasses add the HTTP transport layer (sync or async). +""" + +from __future__ import annotations + +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from ..core.errors import HttpError, ValidationError +from ..core._error_codes import _http_subcode +from ..models.batch import BatchItemResponse, BatchResult +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from ..models.upsert import UpsertItem +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from ._raw_request import _RawRequest +from ._odata_base import _GUID_RE + +if TYPE_CHECKING: + from ._odata_base import _ODataBase + +__all__ = [] + +_CRLF = "\r\n" +_MAX_BATCH_SIZE = 1000 + + +# --------------------------------------------------------------------------- +# Intent dataclasses — one per supported operation type +# (stored at batch-build time; resolved to _RawRequest at execute() time) +# --------------------------------------------------------------------------- + +# --- Record intent types --- + + +@dataclass +class _RecordCreate: + table: str + data: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset items + + +@dataclass +class _RecordUpdate: + table: str + ids: Union[str, List[str]] + changes: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset single-record updates + + +@dataclass +class _RecordDelete: + table: str + ids: Union[str, List[str]] + use_bulk_delete: bool = True + content_id: Optional[int] = None # set only for changeset single-record deletes + + +@dataclass +class _RecordGet: + table: str + record_id: str + select: Optional[List[str]] = None + expand: Optional[List[str]] = None + include_annotations: Optional[str] = None + + +@dataclass +class _RecordList: + table: str + select: Optional[List[str]] = None + filter: Optional[str] = None + orderby: Optional[List[str]] = None + top: Optional[int] = None + expand: Optional[List[str]] = None + page_size: Optional[int] = None + count: bool = False + include_annotations: Optional[str] = None + + +@dataclass +class _RecordUpsert: + table: str + items: List[UpsertItem] # always non-empty; normalised by BatchRecordOperations + + +# --- Table intent types --- + + +@dataclass +class _TableCreate: + table: str + columns: Dict[str, Any] + solution: Optional[str] = None + primary_column: Optional[str] = None + display_name: Optional[str] = None + + +@dataclass +class _TableDelete: + table: str + + +@dataclass +class _TableGet: + table: str + + +@dataclass +class _TableList: + filter: Optional[str] = None + select: Optional[List[str]] = None + + +@dataclass +class _TableAddColumns: + table: str + columns: Dict[str, Any] + + +@dataclass +class _TableRemoveColumns: + table: str + columns: Union[str, List[str]] + + +@dataclass +class _TableCreateOneToMany: + lookup: LookupAttributeMetadata + relationship: OneToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableCreateManyToMany: + relationship: ManyToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableDeleteRelationship: + relationship_id: str + + +@dataclass +class _TableGetRelationship: + schema_name: str + + +@dataclass +class _TableCreateLookupField: + referencing_table: str + lookup_field_name: str + referenced_table: str + display_name: Optional[str] = None + description: Optional[str] = None + required: bool = False + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK + solution: Optional[str] = None + language_code: int = 1033 + + +# --- Query intent types --- + + +@dataclass +class _QuerySql: + sql: str + + +# --------------------------------------------------------------------------- +# Changeset container +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSet: + """Ordered group of single-record write operations that execute atomically. + + Content-IDs are allocated from ``_counter``, a single-element ``List[int]`` + that is shared across all changesets in the same batch. Passing the same + list object to every ``_ChangeSet`` created by a :class:`BatchRequest` + ensures Content-ID values are unique within the entire batch request, not + just within an individual changeset, as required by the OData spec. + + When constructed in isolation (e.g. in unit tests), ``_counter`` defaults + to a fresh ``[1]`` so the class remains self-contained. + """ + + operations: List[Union[_RecordCreate, _RecordUpdate, _RecordDelete]] = field(default_factory=list) + _counter: List[int] = field(default_factory=lambda: [1], repr=False) + + def add_create(self, table: str, data: Dict[str, Any]) -> str: + """Add a single-record create; return its content-ID reference string.""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordCreate(table=table, data=data, content_id=cid)) + return f"${cid}" + + def add_update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: + """Add a single-record update (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordUpdate(table=table, ids=record_id, changes=changes, content_id=cid)) + + def add_delete(self, table: str, record_id: str) -> None: + """Add a single-record delete (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordDelete(table=table, ids=record_id, content_id=cid)) + + +# --------------------------------------------------------------------------- +# Changeset batch item +# (_RawRequest is imported from ._raw_request — defined there so _odata.py +# can also import it without a circular dependency) +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSetBatchItem: + """A resolved changeset — serialised as a nested multipart/mixed part.""" + + requests: List[_RawRequest] + + +# --------------------------------------------------------------------------- +# Batch base: pure serialisation and pure table resolvers +# --------------------------------------------------------------------------- + + +class _BatchBase: + """Pure-logic base for the Dataverse batch client. + + Provides multipart serialisation, response parsing, and the subset of + intent resolvers that require no I/O. Subclasses must supply ``execute`` + and the I/O-dependent resolvers. + + :param od: The active OData client (provides helpers and HTTP transport). + """ + + def __init__(self, od: "_ODataBase") -> None: + self._od = od + + # ------------------------------------------------------------------ + # Pure table resolvers — delegate to _ODataBase._build_* methods + # ------------------------------------------------------------------ + + def _resolve_table_create(self, op: _TableCreate) -> List[_RawRequest]: + return [self._od._build_create_entity(op.table, op.columns, op.solution, op.primary_column, op.display_name)] + + def _resolve_table_get(self, op: _TableGet) -> List[_RawRequest]: + return [self._od._build_get_entity(op.table)] + + def _resolve_table_list(self, op: _TableList) -> List[_RawRequest]: + return [self._od._build_list_entities(filter=op.filter, select=op.select)] + + def _resolve_table_create_one_to_many(self, op: _TableCreateOneToMany) -> List[_RawRequest]: + body = op.relationship.to_dict() + body["Lookup"] = op.lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + def _resolve_table_create_many_to_many(self, op: _TableCreateManyToMany) -> List[_RawRequest]: + return [self._od._build_create_relationship(op.relationship.to_dict(), solution=op.solution)] + + def _resolve_table_delete_relationship(self, op: _TableDeleteRelationship) -> List[_RawRequest]: + return [self._od._build_delete_relationship(op.relationship_id)] + + def _resolve_table_get_relationship(self, op: _TableGetRelationship) -> List[_RawRequest]: + return [self._od._build_get_relationship(op.schema_name)] + + def _resolve_table_create_lookup_field(self, op: _TableCreateLookupField) -> List[_RawRequest]: + lookup, relationship = self._od._build_lookup_field_models( + referencing_table=op.referencing_table, + lookup_field_name=op.lookup_field_name, + referenced_table=op.referenced_table, + display_name=op.display_name, + description=op.description, + required=op.required, + cascade_delete=op.cascade_delete, + language_code=op.language_code, + ) + body = relationship.to_dict() + body["Lookup"] = lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + # ------------------------------------------------------------------ + # Multipart serialisation + # ------------------------------------------------------------------ + + def _build_batch_body( + self, + resolved: List[Union[_RawRequest, _ChangeSetBatchItem]], + batch_boundary: str, + ) -> str: + parts: List[str] = [] + for item in resolved: + if isinstance(item, _ChangeSetBatchItem): + parts.append(self._serialize_changeset_item(item, batch_boundary)) + else: + parts.append(self._serialize_raw_request(item, batch_boundary)) + return "".join(parts) + f"--{batch_boundary}--{_CRLF}" + + def _serialize_raw_request(self, req: _RawRequest, boundary: str) -> str: + """Serialise a single operation as a multipart/mixed part with CRLF line endings.""" + part_header_lines = [ + f"--{boundary}", + "Content-Type: application/http", + "Content-Transfer-Encoding: binary", + ] + if req.content_id is not None: + part_header_lines.append(f"Content-ID: {req.content_id}") + + inner_lines = [f"{req.method} {req.url} HTTP/1.1"] + if req.body is not None: + inner_lines.append("Content-Type: application/json; type=entry") + if req.headers: + for k, v in req.headers.items(): + inner_lines.append(f"{k}: {v}") + inner_lines.append("") # blank line — end of inner headers + if req.body is not None: + inner_lines.append(req.body) + + part_header_str = _CRLF.join(part_header_lines) + _CRLF + inner_str = _CRLF.join(inner_lines) + return part_header_str + _CRLF + inner_str + _CRLF + + def _serialize_changeset_item(self, cs: _ChangeSetBatchItem, batch_boundary: str) -> str: + cs_boundary = f"changeset_{uuid.uuid4()}" + cs_parts = [self._serialize_raw_request(r, cs_boundary) for r in cs.requests] + cs_parts.append(f"--{cs_boundary}--{_CRLF}") + cs_body = "".join(cs_parts) + + outer = ( + f"--{batch_boundary}{_CRLF}" f'Content-Type: multipart/mixed; boundary="{cs_boundary}"{_CRLF}' f"{_CRLF}" + ) + return outer + cs_body + _CRLF + + # ------------------------------------------------------------------ + # Response parsing (multipart/mixed) + # ------------------------------------------------------------------ + + def _parse_batch_response(self, response: Any) -> BatchResult: + content_type = response.headers.get("Content-Type", "") + boundary = _extract_boundary(content_type) + if not boundary: + # Non-multipart response: the batch request itself was rejected by Dataverse + # (common for top-level 4xx, e.g. malformed body, missing OData headers). + # Returning an empty BatchResult() here would silently hide the error and + # make has_errors=False, which is actively misleading. Raise instead. + _raise_top_level_batch_error(response) + return BatchResult() # unreachable; satisfies type checkers + parts = _split_multipart(response.text or "", boundary) + responses: List[BatchItemResponse] = [] + for part_headers, part_body in parts: + part_ct = part_headers.get("content-type", "") + if "multipart/mixed" in part_ct: + inner_boundary = _extract_boundary(part_ct) + if inner_boundary: + for ih, ib in _split_multipart(part_body, inner_boundary): + item = _parse_http_response_part(ib, ih.get("content-id")) + if item is not None: + responses.append(item) + else: + item = _parse_http_response_part(part_body, content_id=part_headers.get("content-id")) + if item is not None: + responses.append(item) + return BatchResult(responses=responses) + + +# --------------------------------------------------------------------------- +# Multipart parsing helpers +# --------------------------------------------------------------------------- + + +def _raise_top_level_batch_error(response: Any) -> None: + """Parse a non-multipart batch response and raise HttpError with the service message. + + Dataverse returns ``application/json`` with an ``{"error": {...}}`` payload when + it rejects the batch request at the HTTP level (e.g. malformed multipart body, + missing OData headers). This helper surfaces that detail instead of silently + returning an empty ``BatchResult``. + """ + status_code: int = getattr(response, "status_code", 0) + service_error_code: Optional[str] = None + try: + payload = response.json() + error = payload.get("error", {}) + service_error_code = error.get("code") or None + message: str = error.get("message") or response.text or "Unexpected non-multipart response from $batch" + except Exception: + message = (getattr(response, "text", None) or "") or "Unexpected non-multipart response from $batch" + raise HttpError( + message=f"Batch request rejected by Dataverse: {message}", + status_code=status_code, + subcode=_http_subcode(status_code) if status_code else None, + service_error_code=service_error_code, + ) + + +_BOUNDARY_RE = re.compile(r'boundary="?([^";,\s]+)"?', re.IGNORECASE) + + +def _extract_boundary(content_type: str) -> Optional[str]: + m = _BOUNDARY_RE.search(content_type) + return m.group(1) if m else None + + +def _split_multipart(body: str, boundary: str) -> List[Tuple[Dict[str, str], str]]: + delimiter = f"--{boundary}" + parts: List[Tuple[Dict[str, str], str]] = [] + lines = body.replace("\r\n", "\n").split("\n") + current: List[str] = [] + in_part = False + for line in lines: + stripped = line.rstrip("\r") + if stripped == delimiter: + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + current = [] + in_part = True + elif stripped == f"{delimiter}--": + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + break + elif in_part: + current.append(line) + return parts + + +def _parse_mime_part(raw: str) -> Tuple[Dict[str, str], str]: + if "\n\n" in raw: + header_block, body = raw.split("\n\n", 1) + else: + header_block, body = raw, "" + headers: Dict[str, str] = {} + for line in header_block.splitlines(): + if ":" in line: + name, _, value = line.partition(":") + headers[name.strip().lower()] = value.strip() + return headers, body.strip() + + +def _parse_http_response_part(text: str, content_id: Optional[str]) -> Optional[BatchItemResponse]: + lines = text.replace("\r\n", "\n").splitlines() + if not lines: + return None + status_line = "" + idx = 0 + for i, line in enumerate(lines): + if line.startswith("HTTP/"): + status_line = line + idx = i + 1 + break + if not status_line: + return None + parts = status_line.split(" ", 2) + if len(parts) < 2: + return None + try: + status_code = int(parts[1]) + except ValueError: + return None + resp_headers: Dict[str, str] = {} + body_start = idx + for i in range(idx, len(lines)): + if lines[i] == "": + body_start = i + 1 + break + if ":" in lines[i]: + name, _, value = lines[i].partition(":") + resp_headers[name.strip().lower()] = value.strip() + entity_id: Optional[str] = None + odata_id = resp_headers.get("odata-entityid", "") + if odata_id: + m = _GUID_RE.search(odata_id) + if m: + entity_id = m.group(0) + body_text = "\n".join(lines[body_start:]).strip() + data: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + error_code: Optional[str] = None + if body_text: + try: + parsed = json.loads(body_text) + if isinstance(parsed, dict): + err = parsed.get("error") + if isinstance(err, dict): + error_message = err.get("message") + error_code = err.get("code") + else: + data = parsed + except (json.JSONDecodeError, ValueError): + pass + return BatchItemResponse( + status_code=status_code, + content_id=content_id, + entity_id=entity_id, + data=data, + error_message=error_message, + error_code=error_code, + ) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 8b27dd20..2264ccf9 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -12,27 +12,15 @@ from dataclasses import dataclass, field import unicodedata import time -import re import json -import uuid import warnings from datetime import datetime, timezone -import importlib.resources as ir -from contextlib import contextmanager -from contextvars import ContextVar -from urllib.parse import quote as _url_quote, parse_qs, urlparse +from urllib.parse import quote as _url_quote from ..core._http import _HttpClient from ._upload import _FileUploadMixin from ._relationships import _RelationshipOperationsMixin -from ..models.relationship import ( - LookupAttributeMetadata, - OneToManyRelationshipMetadata, - CascadeConfiguration, -) -from ..models.labels import Label, LocalizedLabel -from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK from ..core.errors import * from ._raw_request import _RawRequest from ..core._error_codes import ( @@ -48,120 +36,21 @@ METADATA_TABLE_NOT_FOUND, METADATA_TABLE_ALREADY_EXISTS, METADATA_COLUMN_NOT_FOUND, - VALIDATION_UNSUPPORTED_CACHE_KIND, ) -from .. import __version__ as _SDK_VERSION - -_USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" -_GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") -_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) -_DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) - - -def _extract_pagingcookie(next_link: str) -> Optional[str]: - """Extract the raw pagingcookie value from a SQL ``@odata.nextLink`` URL. - - The Dataverse SQL endpoint has a server-side bug where the pagingcookie - (containing first/last record GUIDs) does not advance between pages even - though ``pagenumber`` increments. Detecting a repeated cookie lets the - pagination loop break instead of looping indefinitely. - - Returns the pagingcookie string if present, or ``None`` if not found. - """ - try: - qs = parse_qs(urlparse(next_link).query) - skiptoken = qs.get("$skiptoken", [None])[0] - if not skiptoken: - return None - # parse_qs already URL-decodes the value once, giving the outer XML with - # pagingcookie still percent-encoded (e.g. pagingcookie="%3ccookie..."). - # A second decode is intentionally omitted: decoding again would turn %22 - # into " inside the cookie XML, breaking the regex and causing every page - # to extract the same truncated prefix regardless of the actual GUIDs. - m = re.search(r'pagingcookie="([^"]+)"', skiptoken) - if m: - return m.group(1) - except Exception: - pass - return None - - -@dataclass -class _RequestContext: - """Structured request context used by ``_request`` to clarify payload and metadata.""" - - method: str - url: str - expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES - headers: Optional[Dict[str, str]] = None - kwargs: Dict[str, Any] = field(default_factory=dict) - - @classmethod - def build( - cls, - method: str, - url: str, - *, - expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, - merge_headers: Optional[Callable[[Optional[Dict[str, str]]], Dict[str, str]]] = None, - **kwargs: Any, - ) -> "_RequestContext": - headers = kwargs.get("headers") - headers = merge_headers(headers) if merge_headers else (headers or {}) - headers.setdefault("x-ms-client-request-id", str(uuid.uuid4())) - headers.setdefault("x-ms-correlation-id", _CALL_SCOPE_CORRELATION_ID.get()) - kwargs["headers"] = headers - return cls( - method=method, - url=url, - expected=expected, - headers=headers, - kwargs=kwargs or {}, - ) +from ._odata_base import ( + _ODataBase, + _GUID_RE, + _extract_pagingcookie, + _USER_AGENT, + _DEFAULT_EXPECTED_STATUSES, + _RequestContext, +) -class _ODataClient(_FileUploadMixin, _RelationshipOperationsMixin): +class _ODataClient(_FileUploadMixin, _RelationshipOperationsMixin, _ODataBase): """Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" - @staticmethod - def _escape_odata_quotes(value: str) -> str: - """Escape single quotes for OData queries (by doubling them).""" - return value.replace("'", "''") - - @staticmethod - def _normalize_cache_key(table_schema_name: str) -> str: - """Normalize table_schema_name to lowercase for case-insensitive cache keys.""" - return table_schema_name.lower() if isinstance(table_schema_name, str) else "" - - @staticmethod - def _lowercase_keys(record: Dict[str, Any]) -> Dict[str, Any]: - """Convert all dictionary keys to lowercase for case-insensitive column names. - - Dataverse LogicalNames for attributes are stored lowercase, but users may - provide PascalCase names (matching SchemaName). This normalizes the input. - - Keys containing ``@odata.`` (e.g. ``new_CustomerId@odata.bind``) are - preserved as-is because the navigation property portion before ``@`` - must retain its original casing (case-sensitive navigation property name). The OData - parser validates ``@odata.bind`` property names **case-sensitively** - against the entity's declared navigation properties, so lowercasing - these keys causes ``400 - undeclared property`` errors. - """ - if not isinstance(record, dict): - return record - return {k.lower() if isinstance(k, str) and "@odata." not in k else k: v for k, v in record.items()} - - @staticmethod - def _lowercase_list(items: Optional[List[str]]) -> Optional[List[str]]: - """Convert all strings in a list to lowercase for case-insensitive column names. - - Used for $select and $orderby parameters where column names must be lowercase. - """ - if not items: - return items - return [item.lower() if isinstance(item, str) else item for item in items] - def __init__( self, auth, @@ -183,22 +72,8 @@ def __init__( :type session: :class:`requests.Session` | ``None`` :raises ValueError: If ``base_url`` is empty after stripping. """ + super().__init__(base_url, config) self.auth = auth - self.base_url = (base_url or "").rstrip("/") - if not self.base_url: - raise ValueError("base_url is required.") - self.api = f"{self.base_url}/api/data/v9.2" - self.config = ( - config - or __import__( - "PowerPlatform.Dataverse.core.config", fromlist=["DataverseConfig"] - ).DataverseConfig.from_env() - ) - self._http_logger = None - if self.config.log_config is not None: - from ..core._http_logger import _HttpLogger - - self._http_logger = _HttpLogger(self.config.log_config) self._http = _HttpClient( retries=self.config.http_retries, backoff=self.config.http_backoff, @@ -206,23 +81,6 @@ def __init__( session=session, logger=self._http_logger, ) - ctx_obj = self.config.operation_context - self._operation_context = ctx_obj.user_agent_context if ctx_obj else None - self._logical_to_entityset_cache: dict[str, str] = {} - # Cache: normalized table_schema_name (lowercase) -> primary id attribute (e.g. accountid) - self._logical_primaryid_cache: dict[str, str] = {} - self._picklist_label_cache: dict[str, dict] = {} - self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL - - @contextmanager - def _call_scope(self): - """Context manager to generate a new correlation id for each SDK call scope.""" - shared_id = str(uuid.uuid4()) - token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) - try: - yield shared_id - finally: - _CALL_SCOPE_CORRELATION_ID.reset(token) def close(self) -> None: """Close the OData client and release resources. @@ -230,14 +88,9 @@ def close(self) -> None: Clears all internal caches and closes the underlying HTTP client. Safe to call multiple times. """ - self._logical_to_entityset_cache.clear() - self._logical_primaryid_cache.clear() - self._picklist_label_cache.clear() + super().close() if self._http is not None: self._http.close() - if self._http_logger is not None: - self._http_logger.close() - self._http_logger = None def _headers(self) -> Dict[str, str]: """Build standard OData headers with bearer auth.""" @@ -421,36 +274,6 @@ def _create_multiple(self, entity_set: str, table_schema_name: str, records: Lis return out return [] - def _build_alternate_key_str(self, alternate_key: Dict[str, Any]) -> str: - """Build an OData alternate key segment from a mapping of key names to values. - - String values are single-quoted and escaped; all other values are rendered as-is. - - :param alternate_key: Mapping of alternate key attribute names to their values. - Must be a non-empty dict with string keys. - :type alternate_key: ``dict[str, Any]`` - - :return: Comma-separated key=value pairs suitable for use in a URL segment. - :rtype: ``str`` - - :raises ValueError: If ``alternate_key`` is empty. - :raises TypeError: If any key in ``alternate_key`` is not a string. - """ - if not alternate_key: - raise ValueError("alternate_key must be a non-empty dict") - bad_keys = [k for k in alternate_key if not isinstance(k, str)] - if bad_keys: - raise TypeError(f"alternate_key keys must be strings; got: {bad_keys!r}") - parts = [] - for k, v in alternate_key.items(): - k_lower = k.lower() if isinstance(k, str) else k - if isinstance(v, str): - v_escaped = self._escape_odata_quotes(v) - parts.append(f"{k_lower}='{v_escaped}'") - else: - parts.append(f"{k_lower}={v}") - return ",".join(parts) - def _upsert( self, entity_set: str, @@ -622,23 +445,6 @@ def _delete_multiple( job_id = body.get("JobId") return job_id - def _format_key(self, key: str) -> str: - k = key.strip() - if k.startswith("(") and k.endswith(")"): - return k - # Escape single quotes in alternate key values - if "=" in k and "'" in k: - - def esc(match): - # match.group(1) is the key, match.group(2) is the value - return f"{match.group(1)}='{self._escape_odata_quotes(match.group(2))}'" - - k = re.sub(r"(\w+)=\'([^\']*)\'", esc, k) - return f"({k})" - if len(k) == 36 and "-" in k: - return f"({k})" - return f"({k})" - def _update(self, table_schema_name: str, key: str, data: Dict[str, Any]) -> None: """Update an existing record by GUID. @@ -811,165 +617,6 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st yield [x for x in items if isinstance(x, dict)] next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") if isinstance(data, dict) else None - # ----------------------- SQL guardrail patterns -------------------- - _SQL_WRITE_RE = re.compile( - r"^\s*(?:INSERT|UPDATE|DELETE|DROP|TRUNCATE|ALTER|CREATE|EXEC|GRANT|REVOKE|BULK)\b", - re.IGNORECASE, - ) - _SQL_COMMENT_RE = re.compile(r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|--[^\n]*", re.DOTALL) - _SQL_LEADING_WILDCARD_RE = re.compile(r"\bLIKE\s+'%[^']", re.IGNORECASE) - _SQL_IMPLICIT_CROSS_JOIN_RE = re.compile( - r"\bFROM\s+[A-Za-z0-9_]+(?:\s+[A-Za-z0-9_]+)?\s*,\s*[A-Za-z0-9_]+", - re.IGNORECASE, - ) - # Server-blocked SQL patterns (save the round-trip by catching early) - _SQL_UNSUPPORTED_JOIN_RE = re.compile( - r"\b(?:CROSS\s+JOIN|RIGHT\s+(?:OUTER\s+)?JOIN|FULL\s+(?:OUTER\s+)?JOIN)\b", - re.IGNORECASE, - ) - _SQL_UNION_RE = re.compile(r"\bUNION\b", re.IGNORECASE) - _SQL_HAVING_RE = re.compile(r"\bHAVING\b", re.IGNORECASE) - _SQL_CTE_RE = re.compile(r"^\s*WITH\b", re.IGNORECASE) - _SQL_SUBQUERY_RE = re.compile( - r"\bIN\s*\(\s*SELECT\b|\bEXISTS\s*\(\s*SELECT\b|\(\s*SELECT\b.*\bFROM\b", - re.IGNORECASE, - ) - # SELECT * is intentionally rejected -- not a technical limitation but a - # deliberate design decision. Wide entities (e.g. account has 307 columns) - # make SELECT * extremely expensive on shared database infrastructure. - # COUNT(*) is NOT matched because COUNT appears before the *. - _SQL_SELECT_STAR_RE = re.compile( - r"\bSELECT\b\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?\*\s", - re.IGNORECASE, - ) - - def _sql_guardrails(self, sql: str) -> str: - """Apply safety guardrails to a SQL query before sending to the server. - - Checks split into two categories: - - **Blocked** (``ValidationError`` -- saves a server round-trip): - - 1. Write statements (INSERT/UPDATE/DELETE/DROP/etc.) - 2. CROSS JOIN, RIGHT JOIN, FULL OUTER JOIN (server rejects these) - 3. UNION / UNION ALL (server rejects) - 4. HAVING clause (server rejects) - 5. CTE / WITH clause (server rejects) - 6. Subqueries -- IN (SELECT ...), EXISTS (SELECT ...) (server rejects) - 7. SELECT * -- intentional design decision, not a technical limitation. - Wide entities make wildcard selects extremely expensive on shared - database infrastructure. ``COUNT(*)`` is not affected. - - **Warned** (``UserWarning`` -- query still executes): - - 8. Leading-wildcard LIKE (full table scan) - 9. Implicit cross join FROM a, b (cartesian product) - - All blocked patterns are also blocked by the server, but catching - them here saves the network round-trip and provides clearer error - messages. To bypass a specific check (e.g., if the server adds - support in the future), all checks are in this single method. - - :param sql: The SQL string (already stripped). - :return: The SQL string (unchanged). - :raises ValidationError: If the SQL contains a blocked pattern. - """ - # --- BLOCKED (save server round-trip) --- - - # 1. Block writes (strip SQL comments first to catch comment-prefixed writes) - sql_no_comments = self._SQL_COMMENT_RE.sub(" ", sql).strip() - if self._SQL_WRITE_RE.search(sql_no_comments): - raise ValidationError( - "SQL endpoint is read-only. Use client.records or " - "client.dataframe for write operations " - "(INSERT/UPDATE/DELETE are not supported).", - subcode=VALIDATION_SQL_WRITE_BLOCKED, - ) - - # 2. Block unsupported JOIN types - m = self._SQL_UNSUPPORTED_JOIN_RE.search(sql) - if m: - raise ValidationError( - f"Unsupported JOIN type: '{m.group(0).strip()}'. " - "Only INNER JOIN and LEFT JOIN are supported by the " - "Dataverse SQL endpoint.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 3. Block UNION - if self._SQL_UNION_RE.search(sql): - raise ValidationError( - "UNION is not supported by the Dataverse SQL endpoint. " - "Execute separate queries and combine results in Python " - "(e.g. pd.concat([df1, df2])).", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 4. Block HAVING - if self._SQL_HAVING_RE.search(sql): - raise ValidationError( - "HAVING is not supported by the Dataverse SQL endpoint. " - "Use WHERE to filter before GROUP BY instead.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 5. Block CTE / WITH - if self._SQL_CTE_RE.search(sql): - raise ValidationError( - "CTE (WITH ... AS) is not supported by the Dataverse SQL " - "endpoint. Use separate queries and combine in Python.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 6. Block subqueries - if self._SQL_SUBQUERY_RE.search(sql): - raise ValidationError( - "Subqueries are not supported by the Dataverse SQL " - "endpoint. Use separate SQL calls and combine results " - "in Python (e.g. step 1: get IDs, step 2: WHERE IN).", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 7. Block SELECT * -- intentional design decision. - # Wide entities (e.g. account has 307 columns) make wildcard selects - # extremely expensive on shared database infrastructure. - # COUNT(*) is NOT matched: _SQL_SELECT_STAR_RE requires * to be the - # first token after SELECT/DISTINCT/TOP N, so COUNT appears before *. - if self._SQL_SELECT_STAR_RE.search(sql): - raise ValidationError( - "SELECT * is not supported. Specify column names explicitly " - "(e.g. SELECT name, revenue FROM account). " - "Use client.query.sql_columns('account') to discover available columns.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # --- WARNED (query still executes) --- - - # 8. Warn on leading-wildcard LIKE - if self._SQL_LEADING_WILDCARD_RE.search(sql): - warnings.warn( - "Query contains a leading-wildcard LIKE pattern " - "(e.g. LIKE '%value'). This forces a full table scan " - "and may degrade performance on large tables. " - "Prefer trailing wildcards (LIKE 'value%') when possible.", - UserWarning, - stacklevel=4, - ) - - # 9. Warn on implicit cross joins (server allows but risky) - if self._SQL_IMPLICIT_CROSS_JOIN_RE.search(sql): - warnings.warn( - "Query uses an implicit cross join (FROM table1, table2). " - "This produces a cartesian product that can generate " - "millions of intermediate rows and degrade shared database " - "performance. Use explicit JOIN...ON syntax instead: " - "FROM table1 a JOIN table2 b ON a.column = b.column", - UserWarning, - stacklevel=4, - ) - - return sql - # --------------------------- SQL Custom API ------------------------- def _query_sql(self, sql: str) -> list[dict[str, Any]]: """Execute a read-only SQL SELECT using the Dataverse Web API ``?sql=`` capability. @@ -1087,25 +734,6 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]: return results - @staticmethod - def _extract_logical_table(sql: str) -> str: - """Extract the logical table name after the first standalone FROM. - - Examples: - SELECT * FROM account - SELECT col1, startfrom FROM new_sampleitem WHERE col1 = 1 - - """ - if not isinstance(sql, str): - raise ValueError("sql must be a string") - # Mask out single-quoted string literals to avoid matching FROM inside them. - masked = re.sub(r"'([^']|'')*'", "'x'", sql) - pattern = r"\bfrom\b\s+([A-Za-z0-9_]+)" # minimal, single-line regex - m = re.search(pattern, masked, flags=re.IGNORECASE) - if not m: - raise ValueError("Unable to determine table logical name from SQL (expected 'FROM ').") - return m.group(1).lower() - # ---------------------- Entity set resolution ----------------------- def _entity_set_from_schema_name(self, table_schema_name: str) -> str: """Resolve entity set name (plural) from a schema name (singular) name using metadata. @@ -1158,23 +786,6 @@ def _entity_set_from_schema_name(self, table_schema_name: str) -> str: return es # ---------------------- Table metadata helpers ---------------------- - def _label(self, text: str) -> Dict[str, Any]: - lang = int(self.config.language_code) - return { - "@odata.type": "Microsoft.Dynamics.CRM.Label", - "LocalizedLabels": [ - { - "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", - "Label": text, - "LanguageCode": lang, - } - ], - } - - def _to_pascal(self, name: str) -> str: - parts = re.split(r"[^A-Za-z0-9]+", name) - return "".join(p[:1].upper() + p[1:] for p in parts if p) - def _get_entity_by_table_schema_name( self, table_schema_name: str, @@ -1348,136 +959,6 @@ def _wait_for_attribute_visibility( f"after {total_wait} seconds (exhausted all retries)." ) from last_error - # ---------------------- Enum / Option Set helpers ------------------ - def _build_localizedlabels_payload(self, translations: Dict[int, str]) -> Dict[str, Any]: - """Build a Dataverse Label object from {: } entries. - - Ensures at least one localized label. Does not deduplicate language codes; last wins. - """ - locs: List[Dict[str, Any]] = [] - for lang, text in translations.items(): - if not isinstance(lang, int): - raise ValueError(f"Language code '{lang}' must be int") - if not isinstance(text, str) or not text.strip(): - raise ValueError(f"Label for lang {lang} must be non-empty string") - locs.append( - { - "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", - "Label": text, - "LanguageCode": lang, - } - ) - if not locs: - raise ValueError("At least one translation required") - return { - "@odata.type": "Microsoft.Dynamics.CRM.Label", - "LocalizedLabels": locs, - } - - def _enum_optionset_payload( - self, column_schema_name: str, enum_cls: type[Enum], is_primary_name: bool = False - ) -> Dict[str, Any]: - """Create local (IsGlobal=False) PicklistAttributeMetadata from an Enum subclass. - - Supports translation mapping via optional class attribute `__labels__`: - __labels__ = { 1033: { "Active": "Active", "Inactive": "Inactive" }, - 1036: { "Active": "Actif", "Inactive": "Inactif" } } - - Keys inside per-language dict may be either enum member objects or their names. - If a language lacks a label for a member, member.name is used as fallback. - The client's configured language code is always ensured to exist. - """ - all_member_items = list(enum_cls.__members__.items()) - if not all_member_items: - raise ValueError(f"Enum {enum_cls.__name__} has no members") - - # Duplicate detection - value_to_first_name: Dict[int, str] = {} - for name, member in all_member_items: - val = getattr(member, "value", None) - # Defer non-int validation to later loop for consistency - if val in value_to_first_name and value_to_first_name[val] != name: - raise ValueError( - f"Duplicate enum value {val} in {enum_cls.__name__} (names: {value_to_first_name[val]}, {name})" - ) - value_to_first_name[val] = name - - members = list(enum_cls) - # Validate integer values - for m in members: - if not isinstance(m.value, int): - raise ValueError(f"Enum member '{m.name}' has non-int value '{m.value}' (only int values supported)") - - raw_labels = getattr(enum_cls, "__labels__", None) - labels_by_lang: Dict[int, Dict[str, str]] = {} - if raw_labels is not None: - if not isinstance(raw_labels, dict): - raise ValueError("__labels__ must be a dict {lang:int -> {member: label}}") - # Build a helper map for value -> member name to resolve raw int keys - value_to_name = {m.value: m.name for m in members} - for lang, mapping in raw_labels.items(): - if not isinstance(lang, int): - raise ValueError("Language codes in __labels__ must be ints") - if not isinstance(mapping, dict): - raise ValueError(f"__labels__[{lang}] must be a dict of member names to strings") - labels_by_lang.setdefault(lang, {}) - for k, v in mapping.items(): - # Accept enum member object, its name, or raw int value (from class body reference) - if isinstance(k, enum_cls): - member_name = k.name - elif isinstance(k, int): - member_name = value_to_name.get(k) - if member_name is None: - raise ValueError(f"__labels__[{lang}] has int key {k} not matching any enum value") - else: - member_name = str(k) - if not isinstance(v, str) or not v.strip(): - raise ValueError(f"Label for {member_name} lang {lang} must be non-empty string") - labels_by_lang[lang][member_name] = v - - config_lang = int(self.config.language_code) - # Ensure config language appears (fallback to names) - all_langs = set(labels_by_lang.keys()) | {config_lang} - - options: List[Dict[str, Any]] = [] - for m in sorted(members, key=lambda x: x.value): - per_lang: Dict[int, str] = {} - for lang in all_langs: - label_text = labels_by_lang.get(lang, {}).get(m.name, m.name) - per_lang[lang] = label_text - options.append( - { - "@odata.type": "Microsoft.Dynamics.CRM.OptionMetadata", - "Value": m.value, - "Label": self._build_localizedlabels_payload(per_lang), - } - ) - - attr_label = column_schema_name.split("_")[-1] - return { - "@odata.type": "Microsoft.Dynamics.CRM.PicklistAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(attr_label), - "RequiredLevel": {"Value": "None"}, - "IsPrimaryName": bool(is_primary_name), - "OptionSet": { - "@odata.type": "Microsoft.Dynamics.CRM.OptionSetMetadata", - "IsGlobal": False, - "Options": options, - }, - } - - def _normalize_picklist_label(self, label: str) -> str: - """Normalize a label for case / diacritic insensitive comparison.""" - if not isinstance(label, str): - return "" - # Strip accents - norm = unicodedata.normalize("NFD", label) - norm = "".join(c for c in norm if unicodedata.category(c) != "Mn") - # Collapse whitespace, lowercase - norm = re.sub(r"\s+", " ", norm).strip().lower() - return norm - def _request_metadata_with_retry(self, method: str, url: str, **kwargs): """Fetch metadata with retries on transient errors.""" max_attempts = 5 @@ -1591,105 +1072,6 @@ def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any] resolved_record[k] = val return resolved_record - def _attribute_payload( - self, column_schema_name: str, dtype: Any, *, is_primary_name: bool = False - ) -> Optional[Dict[str, Any]]: - # Enum-based local option set support - if isinstance(dtype, type) and issubclass(dtype, Enum): - return self._enum_optionset_payload(column_schema_name, dtype, is_primary_name=is_primary_name) - if not isinstance(dtype, str): - raise ValueError( - f"Unsupported column spec type for '{column_schema_name}': {type(dtype)} (expected str or Enum subclass)" - ) - dtype_l = dtype.lower().strip() - label = column_schema_name.split("_")[-1] - if dtype_l in ("string", "text"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.StringAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MaxLength": 200, - "FormatName": {"Value": "Text"}, - "IsPrimaryName": bool(is_primary_name), - } - if dtype_l in ("memo", "multiline"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.MemoAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MaxLength": 4000, - "FormatName": {"Value": "Text"}, - "ImeMode": "Auto", - } - if dtype_l in ("int", "integer"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.IntegerAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "Format": "None", - "MinValue": -2147483648, - "MaxValue": 2147483647, - } - if dtype_l in ("decimal", "money"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DecimalAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MinValue": -100000000000.0, - "MaxValue": 100000000000.0, - "Precision": 2, - } - if dtype_l in ("float", "double"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DoubleAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MinValue": -100000000000.0, - "MaxValue": 100000000000.0, - "Precision": 2, - } - if dtype_l in ("datetime", "date"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "Format": "DateOnly", - "ImeMode": "Inactive", - } - if dtype_l in ("bool", "boolean"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.BooleanAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "OptionSet": { - "@odata.type": "Microsoft.Dynamics.CRM.BooleanOptionSetMetadata", - "TrueOption": { - "Value": 1, - "Label": self._label("True"), - }, - "FalseOption": { - "Value": 0, - "Label": self._label("False"), - }, - "IsGlobal": False, - }, - } - if dtype_l == "file": - return { - "@odata.type": "Microsoft.Dynamics.CRM.FileAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - } - return None - def _get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: """Return basic metadata for a custom table if it exists. @@ -2388,210 +1770,6 @@ def _build_list( headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None return _RawRequest(method="GET", url=url, headers=headers) - def _build_create_entity( - self, - table: str, - columns: Dict[str, Any], - solution: Optional[str] = None, - primary_column: Optional[str] = None, - display_name: Optional[str] = None, - ) -> _RawRequest: - """Build an EntityDefinitions POST request without sending it.""" - if primary_column: - primary_attr = primary_column - else: - primary_attr = f"{table.split('_', 1)[0]}_Name" if "_" in table else "new_Name" - attributes = [self._attribute_payload(primary_attr, "string", is_primary_name=True)] - for col_name, dtype in columns.items(): - attr = self._attribute_payload(col_name, dtype) - if not attr: - raise ValidationError( - f"Unsupported column type '{dtype}' for column '{col_name}'.", - subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, - ) - attributes.append(attr) - if display_name is not None: - if not isinstance(display_name, str) or not display_name.strip(): - raise TypeError("display_name must be a non-empty string when provided") - label = display_name if display_name is not None else table - body = { - "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", - "SchemaName": table, - "DisplayName": self._label(label), - "DisplayCollectionName": self._label(label + "s"), - "Description": self._label(f"Custom entity for {label}"), - "OwnershipType": "UserOwned", - "HasActivities": False, - "HasNotes": True, - "IsActivity": False, - "Attributes": attributes, - } - url = f"{self.api}/EntityDefinitions" - if solution: - url += f"?SolutionUniqueName={solution}" - return _RawRequest( - method="POST", - url=url, - body=json.dumps(body, ensure_ascii=False), - ) - - def _build_delete_entity(self, metadata_id: str) -> _RawRequest: - """Build an EntityDefinitions DELETE request without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/EntityDefinitions({metadata_id})", - headers={"If-Match": "*"}, - ) - - def _build_get_entity(self, table: str) -> _RawRequest: - """Build an EntityDefinitions GET request without sending it.""" - logical = self._escape_odata_quotes(table.lower()) - return _RawRequest( - method="GET", - url=( - f"{self.api}/EntityDefinitions" - f"?$select=MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute" - f"&$filter=LogicalName eq '{logical}'" - ), - ) - - def _build_list_entities( - self, - *, - filter: Optional[str] = None, - select: Optional[List[str]] = None, - ) -> _RawRequest: - """Build an EntityDefinitions list GET request without sending it.""" - base_filter = "IsPrivate eq false" - if filter: - combined_filter = f"{base_filter} and ({filter})" - else: - combined_filter = base_filter - url = f"{self.api}/EntityDefinitions?$filter={combined_filter}" - if select is not None and isinstance(select, str): - raise TypeError("select must be a list of property names, not a bare string") - if select: - url += "&$select=" + ",".join(select) - return _RawRequest(method="GET", url=url) - - def _build_create_column( - self, - entity_metadata_id: str, - col_name: str, - dtype: Any, - ) -> _RawRequest: - """Build an Attributes POST request for one column without sending it.""" - attr = self._attribute_payload(col_name, dtype) - if not attr: - raise ValidationError( - f"Unsupported column type '{dtype}' for column '{col_name}'.", - subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, - ) - return _RawRequest( - method="POST", - url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes", - body=json.dumps(attr, ensure_ascii=False), - ) - - def _build_delete_column( - self, - entity_metadata_id: str, - col_metadata_id: str, - ) -> _RawRequest: - """Build an Attributes DELETE request for one column without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes({col_metadata_id})", - headers={"If-Match": "*"}, - ) - - @staticmethod - def _build_lookup_field_models( - referencing_table: str, - lookup_field_name: str, - referenced_table: str, - *, - display_name: Optional[str] = None, - description: Optional[str] = None, - required: bool = False, - cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, - language_code: int = 1033, - ) -> tuple: - """Build a (lookup, relationship) pair for a lookup field creation. - - Returns ``(LookupAttributeMetadata, OneToManyRelationshipMetadata)``. - Used by both the batch resolver and ``TableOperations.create_lookup_field`` - to avoid duplicating the metadata assembly logic. - - Note: ``referencing_table`` and ``referenced_table`` are lowercased - automatically because Dataverse stores entity logical names in - lowercase. ``lookup_field_name`` is kept as-is (it is a SchemaName). - """ - # Dataverse logical names are always lowercase. Callers may pass - # SchemaName-cased values (e.g. "new_SQLTeam"); normalise here so - # the relationship metadata uses valid logical names. - referencing_lower = referencing_table.lower() - referenced_lower = referenced_table.lower() - - lookup = LookupAttributeMetadata( - schema_name=lookup_field_name, - display_name=Label( - localized_labels=[ - LocalizedLabel( - label=display_name or referenced_table, - language_code=language_code, - ) - ] - ), - required_level="ApplicationRequired" if required else "None", - ) - if description: - lookup.description = Label( - localized_labels=[LocalizedLabel(label=description, language_code=language_code)] - ) - rel_name = f"{referenced_lower}_{referencing_lower}_{lookup_field_name}" - relationship = OneToManyRelationshipMetadata( - schema_name=rel_name, - referenced_entity=referenced_lower, - referencing_entity=referencing_lower, - referenced_attribute=f"{referenced_lower}id", - cascade_configuration=CascadeConfiguration(delete=cascade_delete), - ) - return lookup, relationship - - def _build_create_relationship( - self, - body: Dict[str, Any], - *, - solution: Optional[str] = None, - ) -> _RawRequest: - """Build a RelationshipDefinitions POST request without sending it.""" - headers: Dict[str, str] = {} - if solution: - headers["MSCRM.SolutionUniqueName"] = solution - return _RawRequest( - method="POST", - url=f"{self.api}/RelationshipDefinitions", - body=json.dumps(body, ensure_ascii=False), - headers=headers or None, - ) - - def _build_delete_relationship(self, relationship_id: str) -> _RawRequest: - """Build a RelationshipDefinitions DELETE request without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/RelationshipDefinitions({relationship_id})", - headers={"If-Match": "*"}, - ) - - def _build_get_relationship(self, schema_name: str) -> _RawRequest: - """Build a RelationshipDefinitions GET request without sending it.""" - escaped = self._escape_odata_quotes(schema_name) - return _RawRequest( - method="GET", - url=f"{self.api}/RelationshipDefinitions?$filter=SchemaName eq '{escaped}'", - ) - def _build_sql(self, sql: str) -> _RawRequest: """Build a SQL query GET request without sending it. @@ -2613,27 +1791,3 @@ def _build_sql(self, sql: str) -> _RawRequest: method="GET", url=f"{self.api}/{entity_set}?sql={_url_quote(sql, safe='')}", ) - - # ---------------------- Cache maintenance ------------------------- - def _flush_cache( - self, - kind, - ) -> int: - """Flush cached client metadata/state. - - :param kind: Cache kind to flush (only ``"picklist"`` supported). - :type kind: ``str`` - :return: Number of cache entries removed. - :rtype: ``int`` - :raises ValidationError: If ``kind`` is unsupported. - """ - k = (kind or "").strip().lower() - if k != "picklist": - raise ValidationError( - f"Unsupported cache kind '{kind}' (only 'picklist' is implemented)", - subcode=VALIDATION_UNSUPPORTED_CACHE_KIND, - ) - - removed = len(self._picklist_label_cache) - self._picklist_label_cache.clear() - return removed diff --git a/src/PowerPlatform/Dataverse/data/_odata_base.py b/src/PowerPlatform/Dataverse/data/_odata_base.py new file mode 100644 index 00000000..34ff7c55 --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_odata_base.py @@ -0,0 +1,936 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared pure-logic base for the Dataverse OData client. Contains no I/O. + +Subclasses add the HTTP transport layer (sync or async) while sharing all +URL construction, payload building, cache helpers, and other stateless logic. +""" + +from __future__ import annotations + +import json +import re +import unicodedata +import uuid +import warnings +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Union +from urllib.parse import parse_qs, urlparse + +from .. import __version__ as _SDK_VERSION + +from ..core.errors import ValidationError +from ..core._error_codes import ( + VALIDATION_UNSUPPORTED_COLUMN_TYPE, + VALIDATION_UNSUPPORTED_CACHE_KIND, + VALIDATION_SQL_WRITE_BLOCKED, + VALIDATION_SQL_UNSUPPORTED_SYNTAX, +) +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + CascadeConfiguration, +) +from ..models.labels import Label, LocalizedLabel +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from ._raw_request import _RawRequest + +__all__ = [] + +_GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) +_USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" +_DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) + + +def _extract_pagingcookie(next_link: str) -> Optional[str]: + """Extract the raw pagingcookie value from a SQL ``@odata.nextLink`` URL. + + The Dataverse SQL endpoint has a server-side bug where the pagingcookie + (containing first/last record GUIDs) does not advance between pages even + though ``pagenumber`` increments. Detecting a repeated cookie lets the + pagination loop break instead of looping indefinitely. + + Returns the pagingcookie string if present, or ``None`` if not found. + """ + try: + qs = parse_qs(urlparse(next_link).query) + skiptoken = qs.get("$skiptoken", [None])[0] + if not skiptoken: + return None + # parse_qs already URL-decodes the value once, giving the outer XML with + # pagingcookie still percent-encoded (e.g. pagingcookie="%3ccookie..."). + # A second decode is intentionally omitted: decoding again would turn %22 + # into " inside the cookie XML, breaking the regex and causing every page + # to extract the same truncated prefix regardless of the actual GUIDs. + m = re.search(r'pagingcookie="([^"]+)"', skiptoken) + if m: + return m.group(1) + except Exception: + pass + return None + + +@dataclass +class _RequestContext: + """Structured request context used by ``_request`` to clarify payload and metadata.""" + + method: str + url: str + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES + headers: Optional[Dict[str, str]] = None + kwargs: Dict[str, Any] = field(default_factory=dict) + + @classmethod + def build( + cls, + method: str, + url: str, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + merge_headers: Optional[Callable[[Optional[Dict[str, str]]], Dict[str, str]]] = None, + **kwargs: Any, + ) -> "_RequestContext": + headers = kwargs.get("headers") + headers = merge_headers(headers) if merge_headers else (headers or {}) + headers.setdefault("x-ms-client-request-id", str(uuid.uuid4())) + headers.setdefault("x-ms-correlation-id", _CALL_SCOPE_CORRELATION_ID.get()) + kwargs["headers"] = headers + return cls( + method=method, + url=url, + expected=expected, + headers=headers, + kwargs=kwargs or {}, + ) + + +class _ODataBase: + """Pure-logic base for the Dataverse OData client. + + Provides URL construction, cache management, payload builders, and other + stateless or cache-only helpers. No I/O is performed here; subclasses + must supply ``_request`` and the rest of the HTTP transport layer. + """ + + def __init__(self, base_url: str, config=None) -> None: + """Initialise shared state: URL, API root, config, in-memory caches, and HTTP logger. + + :param base_url: Organisation base URL (e.g. ``"https://.crm.dynamics.com"``). + :type base_url: :class:`str` + :param config: Optional Dataverse configuration (HTTP retry, backoff, timeout, + language code, HTTP diagnostic logging). If omitted, ``DataverseConfig.from_env()`` is used. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig or None + :raises ValueError: If ``base_url`` is empty after stripping. + """ + self.base_url = (base_url or "").rstrip("/") + if not self.base_url: + raise ValueError("base_url is required.") + self.api = f"{self.base_url}/api/data/v9.2" + self.config = ( + config + or __import__( + "PowerPlatform.Dataverse.core.config", fromlist=["DataverseConfig"] + ).DataverseConfig.from_env() + ) + self._logical_to_entityset_cache: dict[str, str] = {} + # Cache: normalized table_schema_name (lowercase) -> primary id attribute (e.g. accountid) + self._logical_primaryid_cache: dict[str, str] = {} + self._picklist_label_cache: dict[str, dict] = {} + self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL + ctx_obj = self.config.operation_context + self._operation_context: Optional[str] = ctx_obj.user_agent_context if ctx_obj else None + self._http_logger = None + if self.config.log_config is not None: + from ..core._http_logger import _HttpLogger + + self._http_logger = _HttpLogger(self.config.log_config) + + # ------------------------------------------------------------------ + # Static helpers + # ------------------------------------------------------------------ + + @staticmethod + def _escape_odata_quotes(value: str) -> str: + """Escape single quotes for OData queries (by doubling them).""" + return value.replace("'", "''") + + @staticmethod + def _normalize_cache_key(table_schema_name: str) -> str: + """Normalize table_schema_name to lowercase for case-insensitive cache keys.""" + return table_schema_name.lower() if isinstance(table_schema_name, str) else "" + + @staticmethod + def _lowercase_keys(record: Dict[str, Any]) -> Dict[str, Any]: + """Convert all dictionary keys to lowercase for case-insensitive column names. + + Dataverse LogicalNames for attributes are stored lowercase, but users may + provide PascalCase names (matching SchemaName). This normalizes the input. + + Keys containing ``@odata.`` (e.g. ``new_CustomerId@odata.bind``) are + preserved as-is because the navigation property portion before ``@`` + must retain its original casing (case-sensitive navigation property name). The OData + parser validates ``@odata.bind`` property names **case-sensitively** + against the entity's declared navigation properties, so lowercasing + these keys causes ``400 - undeclared property`` errors. + """ + if not isinstance(record, dict): + return record + return {k.lower() if isinstance(k, str) and "@odata." not in k else k: v for k, v in record.items()} + + @staticmethod + def _lowercase_list(items: Optional[List[str]]) -> Optional[List[str]]: + """Convert all strings in a list to lowercase for case-insensitive column names. + + Used for $select and $orderby parameters where column names must be lowercase. + """ + if not items: + return items + return [item.lower() if isinstance(item, str) else item for item in items] + + @staticmethod + def _extract_logical_table(sql: str) -> str: + """Extract the logical table name after the first standalone FROM. + + Examples: + SELECT * FROM account + SELECT col1, startfrom FROM new_sampleitem WHERE col1 = 1 + + """ + if not isinstance(sql, str): + raise ValueError("sql must be a string") + # Mask out single-quoted string literals to avoid matching FROM inside them. + masked = re.sub(r"'([^']|'')*'", "'x'", sql) + pattern = r"\bfrom\b\s+([A-Za-z0-9_]+)" # minimal, single-line regex + m = re.search(pattern, masked, flags=re.IGNORECASE) + if not m: + raise ValueError("Unable to determine table logical name from SQL (expected 'FROM ').") + return m.group(1).lower() + + # ------------------------------------------------------------------ + # Instance helpers + # ------------------------------------------------------------------ + + @contextmanager + def _call_scope(self): + """Context manager to generate a new correlation id for each SDK call scope.""" + shared_id = str(uuid.uuid4()) + token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) + try: + yield shared_id + finally: + _CALL_SCOPE_CORRELATION_ID.reset(token) + + def _format_key(self, key: str) -> str: + k = key.strip() + if k.startswith("(") and k.endswith(")"): + return k + # Escape single quotes in alternate key values + if "=" in k and "'" in k: + + def esc(match): + # match.group(1) is the key, match.group(2) is the value + return f"{match.group(1)}='{self._escape_odata_quotes(match.group(2))}'" + + k = re.sub(r"(\w+)=\'([^\']*)\'", esc, k) + return f"({k})" + if len(k) == 36 and "-" in k: + return f"({k})" + return f"({k})" + + def _build_alternate_key_str(self, alternate_key: Dict[str, Any]) -> str: + """Build an OData alternate key segment from a mapping of key names to values. + + String values are single-quoted and escaped; all other values are rendered as-is. + + :param alternate_key: Mapping of alternate key attribute names to their values. + Must be a non-empty dict with string keys. + :type alternate_key: ``dict[str, Any]`` + + :return: Comma-separated key=value pairs suitable for use in a URL segment. + :rtype: ``str`` + + :raises ValueError: If ``alternate_key`` is empty. + :raises TypeError: If any key in ``alternate_key`` is not a string. + """ + if not alternate_key: + raise ValueError("alternate_key must be a non-empty dict") + bad_keys = [k for k in alternate_key if not isinstance(k, str)] + if bad_keys: + raise TypeError(f"alternate_key keys must be strings; got: {bad_keys!r}") + parts = [] + for k, v in alternate_key.items(): + k_lower = k.lower() if isinstance(k, str) else k + if isinstance(v, str): + v_escaped = self._escape_odata_quotes(v) + parts.append(f"{k_lower}='{v_escaped}'") + else: + parts.append(f"{k_lower}={v}") + return ",".join(parts) + + def _label(self, text: str) -> Dict[str, Any]: + lang = int(self.config.language_code) + return { + "@odata.type": "Microsoft.Dynamics.CRM.Label", + "LocalizedLabels": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", + "Label": text, + "LanguageCode": lang, + } + ], + } + + def _to_pascal(self, name: str) -> str: + parts = re.split(r"[^A-Za-z0-9]+", name) + return "".join(p[:1].upper() + p[1:] for p in parts if p) + + def _normalize_picklist_label(self, label: str) -> str: + """Normalize a label for case / diacritic insensitive comparison.""" + if not isinstance(label, str): + return "" + # Strip accents + norm = unicodedata.normalize("NFD", label) + norm = "".join(c for c in norm if unicodedata.category(c) != "Mn") + # Collapse whitespace, lowercase + norm = re.sub(r"\s+", " ", norm).strip().lower() + return norm + + # ------------------------------------------------------------------ + # Payload builders (no I/O) + # ------------------------------------------------------------------ + + def _build_localizedlabels_payload(self, translations: Dict[int, str]) -> Dict[str, Any]: + """Build a Dataverse Label object from {: } entries. + + Ensures at least one localized label. Does not deduplicate language codes; last wins. + """ + locs: List[Dict[str, Any]] = [] + for lang, text in translations.items(): + if not isinstance(lang, int): + raise ValueError(f"Language code '{lang}' must be int") + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"Label for lang {lang} must be non-empty string") + locs.append( + { + "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", + "Label": text, + "LanguageCode": lang, + } + ) + if not locs: + raise ValueError("At least one translation required") + return { + "@odata.type": "Microsoft.Dynamics.CRM.Label", + "LocalizedLabels": locs, + } + + def _enum_optionset_payload( + self, column_schema_name: str, enum_cls: type[Enum], is_primary_name: bool = False + ) -> Dict[str, Any]: + """Create local (IsGlobal=False) PicklistAttributeMetadata from an Enum subclass. + + Supports translation mapping via optional class attribute `__labels__`: + __labels__ = { 1033: { "Active": "Active", "Inactive": "Inactive" }, + 1036: { "Active": "Actif", "Inactive": "Inactif" } } + + Keys inside per-language dict may be either enum member objects or their names. + If a language lacks a label for a member, member.name is used as fallback. + The client's configured language code is always ensured to exist. + """ + all_member_items = list(enum_cls.__members__.items()) + if not all_member_items: + raise ValueError(f"Enum {enum_cls.__name__} has no members") + + # Duplicate detection + value_to_first_name: Dict[int, str] = {} + for name, member in all_member_items: + val = getattr(member, "value", None) + # Defer non-int validation to later loop for consistency + if val in value_to_first_name and value_to_first_name[val] != name: + raise ValueError( + f"Duplicate enum value {val} in {enum_cls.__name__} (names: {value_to_first_name[val]}, {name})" + ) + value_to_first_name[val] = name + + members = list(enum_cls) + # Validate integer values + for m in members: + if not isinstance(m.value, int): + raise ValueError(f"Enum member '{m.name}' has non-int value '{m.value}' (only int values supported)") + + raw_labels = getattr(enum_cls, "__labels__", None) + labels_by_lang: Dict[int, Dict[str, str]] = {} + if raw_labels is not None: + if not isinstance(raw_labels, dict): + raise ValueError("__labels__ must be a dict {lang:int -> {member: label}}") + # Build a helper map for value -> member name to resolve raw int keys + value_to_name = {m.value: m.name for m in members} + for lang, mapping in raw_labels.items(): + if not isinstance(lang, int): + raise ValueError("Language codes in __labels__ must be ints") + if not isinstance(mapping, dict): + raise ValueError(f"__labels__[{lang}] must be a dict of member names to strings") + labels_by_lang.setdefault(lang, {}) + for k, v in mapping.items(): + # Accept enum member object, its name, or raw int value (from class body reference) + if isinstance(k, enum_cls): + member_name = k.name + elif isinstance(k, int): + member_name = value_to_name.get(k) + if member_name is None: + raise ValueError(f"__labels__[{lang}] has int key {k} not matching any enum value") + else: + member_name = str(k) + if not isinstance(v, str) or not v.strip(): + raise ValueError(f"Label for {member_name} lang {lang} must be non-empty string") + labels_by_lang[lang][member_name] = v + + config_lang = int(self.config.language_code) + # Ensure config language appears (fallback to names) + all_langs = set(labels_by_lang.keys()) | {config_lang} + + options: List[Dict[str, Any]] = [] + for m in sorted(members, key=lambda x: x.value): + per_lang: Dict[int, str] = {} + for lang in all_langs: + label_text = labels_by_lang.get(lang, {}).get(m.name, m.name) + per_lang[lang] = label_text + options.append( + { + "@odata.type": "Microsoft.Dynamics.CRM.OptionMetadata", + "Value": m.value, + "Label": self._build_localizedlabels_payload(per_lang), + } + ) + + attr_label = column_schema_name.split("_")[-1] + return { + "@odata.type": "Microsoft.Dynamics.CRM.PicklistAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(attr_label), + "RequiredLevel": {"Value": "None"}, + "IsPrimaryName": bool(is_primary_name), + "OptionSet": { + "@odata.type": "Microsoft.Dynamics.CRM.OptionSetMetadata", + "IsGlobal": False, + "Options": options, + }, + } + + def _attribute_payload( + self, column_schema_name: str, dtype: Any, *, is_primary_name: bool = False + ) -> Optional[Dict[str, Any]]: + # Enum-based local option set support + if isinstance(dtype, type) and issubclass(dtype, Enum): + return self._enum_optionset_payload(column_schema_name, dtype, is_primary_name=is_primary_name) + if not isinstance(dtype, str): + raise ValueError( + f"Unsupported column spec type for '{column_schema_name}': {type(dtype)} (expected str or Enum subclass)" + ) + dtype_l = dtype.lower().strip() + label = column_schema_name.split("_")[-1] + if dtype_l in ("string", "text"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.StringAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MaxLength": 200, + "FormatName": {"Value": "Text"}, + "IsPrimaryName": bool(is_primary_name), + } + if dtype_l in ("memo", "multiline"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.MemoAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MaxLength": 4000, + "FormatName": {"Value": "Text"}, + "ImeMode": "Auto", + } + if dtype_l in ("int", "integer"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.IntegerAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "Format": "None", + "MinValue": -2147483648, + "MaxValue": 2147483647, + } + if dtype_l in ("decimal", "money"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DecimalAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MinValue": -100000000000.0, + "MaxValue": 100000000000.0, + "Precision": 2, + } + if dtype_l in ("float", "double"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DoubleAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MinValue": -100000000000.0, + "MaxValue": 100000000000.0, + "Precision": 2, + } + if dtype_l in ("datetime", "date"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "Format": "DateOnly", + "ImeMode": "Inactive", + } + if dtype_l in ("bool", "boolean"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.BooleanAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "OptionSet": { + "@odata.type": "Microsoft.Dynamics.CRM.BooleanOptionSetMetadata", + "TrueOption": { + "Value": 1, + "Label": self._label("True"), + }, + "FalseOption": { + "Value": 0, + "Label": self._label("False"), + }, + "IsGlobal": False, + }, + } + if dtype_l == "file": + return { + "@odata.type": "Microsoft.Dynamics.CRM.FileAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + } + return None + + # ------------------------------------------------------------------ + # Entity / column / relationship _build_* methods (no I/O) + # ------------------------------------------------------------------ + + def _build_create_entity( + self, + table: str, + columns: Dict[str, Any], + solution: Optional[str] = None, + primary_column: Optional[str] = None, + display_name: Optional[str] = None, + ) -> _RawRequest: + """Build an EntityDefinitions POST request without sending it.""" + if primary_column: + primary_attr = primary_column + else: + primary_attr = f"{table.split('_', 1)[0]}_Name" if "_" in table else "new_Name" + attributes = [self._attribute_payload(primary_attr, "string", is_primary_name=True)] + for col_name, dtype in columns.items(): + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + attributes.append(attr) + if display_name is not None: + if not isinstance(display_name, str) or not display_name.strip(): + raise TypeError("display_name must be a non-empty string when provided") + label = display_name if display_name is not None else table + body = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": table, + "DisplayName": self._label(label), + "DisplayCollectionName": self._label(label + "s"), + "Description": self._label(f"Custom entity for {label}"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsActivity": False, + "Attributes": attributes, + } + url = f"{self.api}/EntityDefinitions" + if solution: + url += f"?SolutionUniqueName={solution}" + return _RawRequest( + method="POST", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + def _build_delete_entity(self, metadata_id: str) -> _RawRequest: + """Build an EntityDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({metadata_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_entity(self, table: str) -> _RawRequest: + """Build an EntityDefinitions GET request without sending it.""" + logical = self._escape_odata_quotes(table.lower()) + return _RawRequest( + method="GET", + url=( + f"{self.api}/EntityDefinitions" + f"?$select=MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute" + f"&$filter=LogicalName eq '{logical}'" + ), + ) + + def _build_list_entities( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> _RawRequest: + """Build an EntityDefinitions list GET request without sending it.""" + base_filter = "IsPrivate eq false" + if filter: + combined_filter = f"{base_filter} and ({filter})" + else: + combined_filter = base_filter + url = f"{self.api}/EntityDefinitions?$filter={combined_filter}" + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + url += "&$select=" + ",".join(select) + return _RawRequest(method="GET", url=url) + + def _build_create_column( + self, + entity_metadata_id: str, + col_name: str, + dtype: Any, + ) -> _RawRequest: + """Build an Attributes POST request for one column without sending it.""" + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + return _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + + def _build_delete_column( + self, + entity_metadata_id: str, + col_metadata_id: str, + ) -> _RawRequest: + """Build an Attributes DELETE request for one column without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes({col_metadata_id})", + headers={"If-Match": "*"}, + ) + + @staticmethod + def _build_lookup_field_models( + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + language_code: int = 1033, + ) -> tuple: + """Build a (lookup, relationship) pair for a lookup field creation. + + Returns ``(LookupAttributeMetadata, OneToManyRelationshipMetadata)``. + Used by both the batch resolver and ``TableOperations.create_lookup_field`` + to avoid duplicating the metadata assembly logic. + + Note: ``referencing_table`` and ``referenced_table`` are lowercased + automatically because Dataverse stores entity logical names in + lowercase. ``lookup_field_name`` is kept as-is (it is a SchemaName). + """ + # Dataverse logical names are always lowercase. Callers may pass + # SchemaName-cased values (e.g. "new_SQLTeam"); normalise here so + # the relationship metadata uses valid logical names. + referencing_lower = referencing_table.lower() + referenced_lower = referenced_table.lower() + + lookup = LookupAttributeMetadata( + schema_name=lookup_field_name, + display_name=Label( + localized_labels=[ + LocalizedLabel( + label=display_name or referenced_table, + language_code=language_code, + ) + ] + ), + required_level="ApplicationRequired" if required else "None", + ) + if description: + lookup.description = Label( + localized_labels=[LocalizedLabel(label=description, language_code=language_code)] + ) + rel_name = f"{referenced_lower}_{referencing_lower}_{lookup_field_name}" + relationship = OneToManyRelationshipMetadata( + schema_name=rel_name, + referenced_entity=referenced_lower, + referencing_entity=referencing_lower, + referenced_attribute=f"{referenced_lower}id", + cascade_configuration=CascadeConfiguration(delete=cascade_delete), + ) + return lookup, relationship + + def _build_create_relationship( + self, + body: Dict[str, Any], + *, + solution: Optional[str] = None, + ) -> _RawRequest: + """Build a RelationshipDefinitions POST request without sending it.""" + headers: Dict[str, str] = {} + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + return _RawRequest( + method="POST", + url=f"{self.api}/RelationshipDefinitions", + body=json.dumps(body, ensure_ascii=False), + headers=headers or None, + ) + + def _build_delete_relationship(self, relationship_id: str) -> _RawRequest: + """Build a RelationshipDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/RelationshipDefinitions({relationship_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_relationship(self, schema_name: str) -> _RawRequest: + """Build a RelationshipDefinitions GET request without sending it.""" + escaped = self._escape_odata_quotes(schema_name) + return _RawRequest( + method="GET", + url=f"{self.api}/RelationshipDefinitions?$filter=SchemaName eq '{escaped}'", + ) + + # ------------------------------------------------------------------ + # SQL guardrails + # ------------------------------------------------------------------ + + # ----------------------- SQL guardrail patterns -------------------- + _SQL_WRITE_RE = re.compile( + r"^\s*(?:INSERT|UPDATE|DELETE|DROP|TRUNCATE|ALTER|CREATE|EXEC|GRANT|REVOKE|BULK)\b", + re.IGNORECASE, + ) + _SQL_COMMENT_RE = re.compile(r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|--[^\n]*", re.DOTALL) + _SQL_LEADING_WILDCARD_RE = re.compile(r"\bLIKE\s+'%[^']", re.IGNORECASE) + _SQL_IMPLICIT_CROSS_JOIN_RE = re.compile( + r"\bFROM\s+[A-Za-z0-9_]+(?:\s+[A-Za-z0-9_]+)?\s*,\s*[A-Za-z0-9_]+", + re.IGNORECASE, + ) + # Server-blocked SQL patterns (save the round-trip by catching early) + _SQL_UNSUPPORTED_JOIN_RE = re.compile( + r"\b(?:CROSS\s+JOIN|RIGHT\s+(?:OUTER\s+)?JOIN|FULL\s+(?:OUTER\s+)?JOIN)\b", + re.IGNORECASE, + ) + _SQL_UNION_RE = re.compile(r"\bUNION\b", re.IGNORECASE) + _SQL_HAVING_RE = re.compile(r"\bHAVING\b", re.IGNORECASE) + _SQL_CTE_RE = re.compile(r"^\s*WITH\b", re.IGNORECASE) + _SQL_SUBQUERY_RE = re.compile( + r"\bIN\s*\(\s*SELECT\b|\bEXISTS\s*\(\s*SELECT\b|\(\s*SELECT\b.*\bFROM\b", + re.IGNORECASE, + ) + # SELECT * is intentionally rejected -- not a technical limitation but a + # deliberate design decision. Wide entities (e.g. account has 307 columns) + # make SELECT * extremely expensive on shared database infrastructure. + # COUNT(*) is NOT matched because COUNT appears before the *. + _SQL_SELECT_STAR_RE = re.compile( + r"\bSELECT\b\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?\*\s", + re.IGNORECASE, + ) + + def _sql_guardrails(self, sql: str) -> str: + """Apply safety guardrails to a SQL query before sending to the server. + + Checks split into two categories: + + **Blocked** (``ValidationError`` -- saves a server round-trip): + + 1. Write statements (INSERT/UPDATE/DELETE/DROP/etc.) + 2. CROSS JOIN, RIGHT JOIN, FULL OUTER JOIN (server rejects these) + 3. UNION / UNION ALL (server rejects) + 4. HAVING clause (server rejects) + 5. CTE / WITH clause (server rejects) + 6. Subqueries -- IN (SELECT ...), EXISTS (SELECT ...) (server rejects) + 7. SELECT * -- intentional design decision, not a technical limitation. + Wide entities make wildcard selects extremely expensive on shared + database infrastructure. ``COUNT(*)`` is not affected. + + **Warned** (``UserWarning`` -- query still executes): + + 8. Leading-wildcard LIKE (full table scan) + 9. Implicit cross join FROM a, b (cartesian product) + + All blocked patterns are also blocked by the server, but catching + them here saves the network round-trip and provides clearer error + messages. To bypass a specific check (e.g., if the server adds + support in the future), all checks are in this single method. + + :param sql: The SQL string (already stripped). + :return: The SQL string (unchanged). + :raises ValidationError: If the SQL contains a blocked pattern. + """ + # --- BLOCKED (save server round-trip) --- + + # 1. Block writes (strip SQL comments first to catch comment-prefixed writes) + sql_no_comments = self._SQL_COMMENT_RE.sub(" ", sql).strip() + if self._SQL_WRITE_RE.search(sql_no_comments): + raise ValidationError( + "SQL endpoint is read-only. Use client.records or " + "client.dataframe for write operations " + "(INSERT/UPDATE/DELETE are not supported).", + subcode=VALIDATION_SQL_WRITE_BLOCKED, + ) + + # 2. Block unsupported JOIN types + m = self._SQL_UNSUPPORTED_JOIN_RE.search(sql) + if m: + raise ValidationError( + f"Unsupported JOIN type: '{m.group(0).strip()}'. " + "Only INNER JOIN and LEFT JOIN are supported by the " + "Dataverse SQL endpoint.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 3. Block UNION + if self._SQL_UNION_RE.search(sql): + raise ValidationError( + "UNION is not supported by the Dataverse SQL endpoint. " + "Execute separate queries and combine results in Python " + "(e.g. pd.concat([df1, df2])).", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 4. Block HAVING + if self._SQL_HAVING_RE.search(sql): + raise ValidationError( + "HAVING is not supported by the Dataverse SQL endpoint. " + "Use WHERE to filter before GROUP BY instead.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 5. Block CTE / WITH + if self._SQL_CTE_RE.search(sql): + raise ValidationError( + "CTE (WITH ... AS) is not supported by the Dataverse SQL " + "endpoint. Use separate queries and combine in Python.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 6. Block subqueries + if self._SQL_SUBQUERY_RE.search(sql): + raise ValidationError( + "Subqueries are not supported by the Dataverse SQL " + "endpoint. Use separate SQL calls and combine results " + "in Python (e.g. step 1: get IDs, step 2: WHERE IN).", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 7. Block SELECT * -- intentional design decision. + # Wide entities (e.g. account has 307 columns) make wildcard selects + # extremely expensive on shared database infrastructure. + # COUNT(*) is NOT matched: _SQL_SELECT_STAR_RE requires * to be the + # first token after SELECT/DISTINCT/TOP N, so COUNT appears before *. + if self._SQL_SELECT_STAR_RE.search(sql): + raise ValidationError( + "SELECT * is not supported. Specify column names explicitly " + "(e.g. SELECT name, revenue FROM account). " + "Use client.query.sql_columns('account') to discover available columns.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # --- WARNED (query still executes) --- + + # 8. Warn on leading-wildcard LIKE + if self._SQL_LEADING_WILDCARD_RE.search(sql): + warnings.warn( + "Query contains a leading-wildcard LIKE pattern " + "(e.g. LIKE '%value'). This forces a full table scan " + "and may degrade performance on large tables. " + "Prefer trailing wildcards (LIKE 'value%') when possible.", + UserWarning, + stacklevel=4, + ) + + # 9. Warn on implicit cross joins (server allows but risky) + if self._SQL_IMPLICIT_CROSS_JOIN_RE.search(sql): + warnings.warn( + "Query uses an implicit cross join (FROM table1, table2). " + "This produces a cartesian product that can generate " + "millions of intermediate rows and degrade shared database " + "performance. Use explicit JOIN...ON syntax instead: " + "FROM table1 a JOIN table2 b ON a.column = b.column", + UserWarning, + stacklevel=4, + ) + + return sql + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def close(self) -> None: + """Clear in-memory caches and close the HTTP diagnostic logger. + + Called by subclass ``close()`` via ``super()``. Safe to call multiple times. + """ + self._logical_to_entityset_cache.clear() + self._logical_primaryid_cache.clear() + self._picklist_label_cache.clear() + if self._http_logger is not None: + self._http_logger.close() + self._http_logger = None + + # ------------------------------------------------------------------ + # Cache maintenance + # ------------------------------------------------------------------ + + def _flush_cache( + self, + kind, + ) -> int: + """Flush cached client metadata/state. + + :param kind: Cache kind to flush (only ``"picklist"`` supported). + :type kind: ``str`` + :return: Number of cache entries removed. + :rtype: ``int`` + :raises ValidationError: If ``kind`` is unsupported. + """ + k = (kind or "").strip().lower() + if k != "picklist": + raise ValidationError( + f"Unsupported cache kind '{kind}' (only 'picklist' is implemented)", + subcode=VALIDATION_UNSUPPORTED_CACHE_KIND, + ) + + removed = len(self._picklist_label_cache) + self._picklist_label_cache.clear() + return removed diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index e7044d2e..bb2664fe 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -49,8 +49,18 @@ from __future__ import annotations +import sys import warnings -from typing import Any, Dict, Iterator, List, Optional, TypedDict, Union +from typing import Any, Iterator, List, Optional, TypedDict, Union + +# typing.Self (PEP 673, Python 3.11+) makes fluent methods return the concrete +# subclass type. TypeVar fallback for Python 3.10 uses the same name so docs render identically. +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing import TypeVar + + Self = TypeVar("Self", bound="_QueryBuilderBase") # type: ignore[assignment] import pandas as pd @@ -171,29 +181,17 @@ def to_odata(self) -> str: return self.relation -class QueryBuilder: - """Fluent interface for building OData queries. - - Provides method chaining for constructing complex queries with - composable filter expressions. Can be used standalone (via :meth:`build`) - or bound to a client (via :meth:`execute`). - - :param table: Table schema name to query. - :type table: str - :raises ValueError: If ``table`` is empty. +class _QueryBuilderBase: + """Pure fluent interface for building OData queries — no I/O. - Example: - Standalone query construction:: + Holds all query state and chaining methods (``select``, ``where``, + ``order_by``, ``top``, ``page_size``, ``count``, ``expand``, + ``include_annotations``, ``include_formatted_values``) and + :meth:`build`. - from PowerPlatform.Dataverse.models.filters import col - - query = (QueryBuilder("account") - .select("name") - .where(col("statecode") == 0) - .top(10)) - params = query.build() - # {"table": "account", "select": ["name"], - # "filter": "statecode eq 0", "top": 10} + Subclasses add execution: :class:`QueryBuilder` for sync clients, + :class:`~PowerPlatform.Dataverse.aio.models.async_query_builder.AsyncQueryBuilder` + for async clients. """ def __init__(self, table: str) -> None: @@ -213,7 +211,7 @@ def __init__(self, table: str) -> None: # ----------------------------------------------------------------- select - def select(self, *columns: str) -> QueryBuilder: + def select(self, *columns: str) -> Self: """Select specific columns to retrieve. Column names are passed as-is; the OData layer lowercases them @@ -231,7 +229,7 @@ def select(self, *columns: str) -> QueryBuilder: # ------------------------------------------------------ filter: expression tree - def where(self, expression: filters.FilterExpression) -> QueryBuilder: + def where(self, expression: filters.FilterExpression) -> Self: """Add a composable filter expression. Accepts a :class:`~PowerPlatform.Dataverse.models.filters.FilterExpression` @@ -260,7 +258,7 @@ def where(self, expression: filters.FilterExpression) -> QueryBuilder: # --------------------------------------------------------------- ordering - def order_by(self, column: str, descending: bool = False) -> QueryBuilder: + def order_by(self, column: str, descending: bool = False) -> Self: """Add sorting order. Can be called multiple times for multi-column sorting. @@ -275,7 +273,7 @@ def order_by(self, column: str, descending: bool = False) -> QueryBuilder: # --------------------------------------------------------------- pagination - def top(self, count: int) -> QueryBuilder: + def top(self, count: int) -> Self: """Limit the total number of results. :param count: Maximum number of records to return (must be >= 1). @@ -287,7 +285,7 @@ def top(self, count: int) -> QueryBuilder: self._top = count return self - def page_size(self, size: int) -> QueryBuilder: + def page_size(self, size: int) -> Self: """Set the number of records per page. Controls how many records are returned in each page/batch @@ -302,7 +300,7 @@ def page_size(self, size: int) -> QueryBuilder: self._page_size = size return self - def count(self) -> QueryBuilder: + def count(self) -> Self: """Request a count of matching records in the response. Adds ``$count=true`` to the query, causing the server to include @@ -321,7 +319,7 @@ def count(self) -> QueryBuilder: self._count = True return self - def include_formatted_values(self) -> QueryBuilder: + def include_formatted_values(self) -> Self: """Request formatted values in the response. Adds ``Prefer: odata.include-annotations="OData.Community.Display.V1.FormattedValue"`` @@ -351,7 +349,7 @@ def include_formatted_values(self) -> QueryBuilder: self._include_annotations = "OData.Community.Display.V1.FormattedValue" return self - def include_annotations(self, annotation: str = "*") -> QueryBuilder: + def include_annotations(self, annotation: str = "*") -> Self: """Request specific OData annotations in the response. Sets the ``Prefer: odata.include-annotations`` header. Use ``"*"`` @@ -379,7 +377,7 @@ def include_annotations(self, annotation: str = "*") -> QueryBuilder: # --------------------------------------------------------------- expand - def expand(self, *relations: Union[str, ExpandOption]) -> QueryBuilder: + def expand(self, *relations: Union[str, ExpandOption]) -> Self: """Expand navigation properties. Accepts plain navigation property names (case-sensitive, passed @@ -448,6 +446,32 @@ def build(self) -> QueryParams: params["include_annotations"] = self._include_annotations return params + +class QueryBuilder(_QueryBuilderBase): + """Fluent interface for building and executing OData queries against a sync client. + + Provides method chaining for constructing complex queries with + composable filter expressions. Can be used standalone (via :meth:`build`) + or bound to a client (via :meth:`execute`). + + :param table: Table schema name to query. + :type table: str + :raises ValueError: If ``table`` is empty. + + Example: + Standalone query construction:: + + from PowerPlatform.Dataverse.models.filters import col + + query = (QueryBuilder("account") + .select("name") + .where(col("statecode") == 0) + .top(10)) + params = query.build() + # {"table": "account", "select": ["name"], + # "filter": "statecode eq 0", "top": 10} + """ + # --------------------------------------------------------------- execute def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterator[QueryResult]]: diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index da65e7a5..e81f412c 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -6,7 +6,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, KeysView, List, Optional, Union, ValuesView, ItemsView +from typing import Any, Dict, Iterator, KeysView, List, Optional, ValuesView, ItemsView __all__ = ["Record", "QueryResult"] @@ -141,7 +141,7 @@ def __bool__(self) -> bool: def __repr__(self) -> str: return f"QueryResult({len(self.records)} records)" - def __getitem__(self, index: Union[int, slice]) -> Union[Record, "QueryResult"]: + def __getitem__(self, index): result = self.records[index] return QueryResult(result) if isinstance(index, slice) else result diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index aa5d8391..062da4ab 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -6,7 +6,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union import pandas as pd @@ -59,6 +59,29 @@ ] +# --------------------------------------------------------------------------- +# Shared interface for batch operation namespaces +# --------------------------------------------------------------------------- + + +class _BatchContext(Protocol): + """Structural interface required by batch operation namespaces. + + The operation namespaces (BatchRecordOperations, BatchTableOperations, etc.) + are pure (no I/O) and shared by both the sync and async batch implementations. + This Protocol allows them to type-annotate their ``batch`` parameter correctly + without importing either concrete class (``BatchRequest`` or + ``AsyncBatchRequest``), which would otherwise require ``# type: ignore``. + + Both :class:`~PowerPlatform.Dataverse.operations.batch.BatchRequest` and + :class:`~PowerPlatform.Dataverse.aio.operations.async_batch.AsyncBatchRequest` + satisfy this protocol structurally — no explicit inheritance needed. + """ + + _items: List[Any] + records: Any # used by BatchDataFrameOperations to delegate create/update/delete + + # --------------------------------------------------------------------------- # Changeset namespaces # --------------------------------------------------------------------------- @@ -180,7 +203,7 @@ class BatchRecordOperations: Do not instantiate directly; use ``batch.records``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create( @@ -482,7 +505,7 @@ class BatchTableOperations: Do not instantiate directly; use ``batch.tables``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create( @@ -720,7 +743,7 @@ class BatchQueryOperations: Do not instantiate directly; use ``batch.query``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def sql(self, sql: str) -> None: @@ -774,7 +797,7 @@ class BatchDataFrameOperations: result = batch.execute() """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create(self, table: str, records: pd.DataFrame) -> None: diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 1f3c8ef2..c927bdf0 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -33,14 +33,12 @@ class QueryOperations: Example:: - from PowerPlatform.Dataverse.models.filters import col - client = DataverseClient(base_url, credential) # Fluent query builder (recommended) for record in (client.query.builder("account") .select("name", "revenue") - .where(col("statecode") == 0) + .filter_eq("statecode", 0) .order_by("revenue", descending=True) .top(100) .execute()): @@ -72,12 +70,10 @@ def builder(self, table: str) -> QueryBuilder: Example: Build and execute a query fluently:: - from PowerPlatform.Dataverse.models.filters import col - for record in (client.query.builder("account") .select("name", "revenue") - .where(col("statecode") == 0) - .where(col("revenue") > 1_000_000) + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -86,11 +82,11 @@ def builder(self, table: str) -> QueryBuilder: With composable expression tree:: - from PowerPlatform.Dataverse.models.filters import col + from PowerPlatform.Dataverse.models.filters import eq, gt for record in (client.query.builder("account") - .where((col("statecode") == 0) | (col("statecode") == 1)) - .where(col("revenue") > 100_000) + .where((eq("statecode", 0) | eq("statecode", 1)) + & gt("revenue", 100000)) .execute()): print(record["name"]) """ diff --git a/tests/unit/aio/__init__.py b/tests/unit/aio/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/unit/aio/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/unit/aio/conftest.py b/tests/unit/aio/conftest.py new file mode 100644 index 00000000..ab29d7c9 --- /dev/null +++ b/tests/unit/aio/conftest.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared fixtures for async unit tests.""" + +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock + +import pytest +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + +@pytest.fixture +def mock_od() -> AsyncMock: + """AsyncMock representing the low-level _AsyncODataClient.""" + od = AsyncMock() + # _call_scope() is a sync context manager; MagicMock supports __enter__/__exit__ + od._call_scope.return_value = MagicMock() + return od + + +@pytest.fixture +def async_client(mock_od: AsyncMock) -> AsyncDataverseClient: + """AsyncDataverseClient with _scoped_odata patched to yield mock_od.""" + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped_odata(): + yield mock_od + + client._scoped_odata = _fake_scoped_odata + return client diff --git a/tests/unit/aio/core/__init__.py b/tests/unit/aio/core/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/unit/aio/core/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/unit/aio/core/test_async_auth.py b/tests/unit/aio/core/test_async_auth.py new file mode 100644 index 00000000..6b425826 --- /dev/null +++ b/tests/unit/aio/core/test_async_auth.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.core._async_auth import _AsyncAuthManager +from PowerPlatform.Dataverse.core._auth import _TokenPair + + +class TestAsyncAuthManager: + """Tests for _AsyncAuthManager credential validation and token acquisition.""" + + def test_non_async_token_credential_raises(self): + """_AsyncAuthManager raises TypeError when credential does not implement AsyncTokenCredential.""" + with pytest.raises(TypeError) as exc_info: + _AsyncAuthManager("not-a-credential") + assert "AsyncTokenCredential" in str(exc_info.value) + + def test_valid_credential_accepted(self): + """_AsyncAuthManager accepts a valid AsyncTokenCredential.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + manager = _AsyncAuthManager(mock_cred) + assert manager.credential is mock_cred + + async def test_acquire_token_returns_token_pair(self): + """_acquire_token calls get_token and returns a _TokenPair with scope and token.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + mock_cred.get_token = AsyncMock(return_value=MagicMock(token="my-access-token")) + + manager = _AsyncAuthManager(mock_cred) + result = await manager._acquire_token("https://org.crm.dynamics.com/.default") + + mock_cred.get_token.assert_called_once_with("https://org.crm.dynamics.com/.default") + assert isinstance(result, _TokenPair) + assert result.resource == "https://org.crm.dynamics.com/.default" + assert result.access_token == "my-access-token" + + async def test_acquire_token_different_scope(self): + """_acquire_token passes the scope string through to get_token.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + mock_cred.get_token = AsyncMock(return_value=MagicMock(token="tok")) + + manager = _AsyncAuthManager(mock_cred) + await manager._acquire_token("https://example.crm10.dynamics.com/.default") + + mock_cred.get_token.assert_called_once_with("https://example.crm10.dynamics.com/.default") diff --git a/tests/unit/aio/core/test_async_http.py b/tests/unit/aio/core/test_async_http.py new file mode 100644 index 00000000..43708aba --- /dev/null +++ b/tests/unit/aio/core/test_async_http.py @@ -0,0 +1,286 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock, call, patch + +import aiohttp + +from PowerPlatform.Dataverse.aio.core._async_http import _AsyncHttpClient, _AsyncResponse + + +def _make_resp(status: int = 200) -> MagicMock: + """Return a mock aiohttp.ClientResponse.""" + resp = MagicMock() + resp.status = status + resp.headers = {} + resp.read = AsyncMock(return_value=b"") + return resp + + +def _make_cm(resp=None, exc=None) -> MagicMock: + """Return an async context manager mock. + + If exc is given, __aenter__ raises it. Otherwise it returns resp. + """ + cm = MagicMock() + if exc is not None: + cm.__aenter__ = AsyncMock(side_effect=exc) + else: + cm.__aenter__ = AsyncMock(return_value=resp) + cm.__aexit__ = AsyncMock(return_value=False) + return cm + + +def _make_session(status: int = 200) -> MagicMock: + """Return a mock aiohttp.ClientSession whose request() is an async context manager.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.request = MagicMock(return_value=_make_cm(_make_resp(status))) + return session + + +class TestAsyncHttpClientTimeout: + """Tests for automatic timeout selection in _AsyncHttpClient._request.""" + + async def test_get_uses_10s_default_timeout(self): + """GET requests use 10 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("get", "https://example.com/data") + _, kwargs = session.request.call_args + assert isinstance(kwargs["timeout"], aiohttp.ClientTimeout) + assert kwargs["timeout"].total == 10 + + async def test_post_uses_120s_default_timeout(self): + """POST requests use 120 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("post", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 120 + + async def test_delete_uses_120s_default_timeout(self): + """DELETE requests use 120 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("delete", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 120 + + async def test_put_uses_10s_default_timeout(self): + """PUT requests use 10 s default (only POST/DELETE get 120 s).""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("put", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 10 + + async def test_patch_uses_10s_default_timeout(self): + """PATCH requests use 10 s default (only POST/DELETE get 120 s).""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("patch", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 10 + + async def test_custom_client_timeout_overrides_method_default(self): + """Explicit default_timeout on the client overrides per-method defaults.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, timeout=30.0, session=session) + await client._request("get", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 30.0 + + async def test_explicit_timeout_kwarg_takes_precedence(self): + """If timeout is already in kwargs it is passed through unchanged.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, timeout=30.0, session=session) + custom_timeout = aiohttp.ClientTimeout(total=5) + await client._request("get", "https://example.com/data", timeout=custom_timeout) + _, kwargs = session.request.call_args + assert kwargs["timeout"] is custom_timeout + + +class TestAsyncHttpClientNoSession: + """Tests for RuntimeError when no session is provided.""" + + async def test_raises_runtime_error_without_session(self): + """_request raises RuntimeError if no session has been set.""" + client = _AsyncHttpClient(retries=1) + with pytest.raises(RuntimeError, match="No aiohttp.ClientSession"): + await client._request("get", "https://example.com") + + +class TestAsyncHttpClientRetry: + """Tests for retry behavior on aiohttp.ClientError.""" + + async def test_retries_on_client_error_and_succeeds(self): + """Retries after a ClientError and returns response on second attempt.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError("timeout")), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=2, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + result = await client._request("get", "https://example.com/data") + + assert session.request.call_count == 2 + assert isinstance(result, _AsyncResponse) + assert result.status == 200 + + async def test_raises_after_all_retries_exhausted(self): + """Raises ClientError after all retry attempts fail.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.request = MagicMock(return_value=_make_cm(exc=aiohttp.ClientConnectionError("timeout"))) + client = _AsyncHttpClient(retries=3, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(aiohttp.ClientError): + await client._request("get", "https://example.com/data") + + async def test_backoff_delay_between_retries(self): + """Sleeps with exponential backoff between retry attempts.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=3, backoff=1.0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await client._request("get", "https://example.com/data") + # First retry: 1.0 * 2^0 = 1.0; second retry: 1.0 * 2^1 = 2.0 + mock_sleep.assert_has_calls([call(1.0), call(2.0)]) + + async def test_no_retry_on_success(self): + """Single successful response does not trigger retries.""" + session = _make_session(200) + client = _AsyncHttpClient(retries=5, backoff=0, session=session) + await client._request("get", "https://example.com/data") + assert session.request.call_count == 1 + + async def test_retries_on_timeout_error(self): + """Retries on asyncio.TimeoutError (not a subclass of aiohttp.ClientError).""" + import asyncio + + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=asyncio.TimeoutError()), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=2, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + result = await client._request("get", "https://example.com/data") + + assert session.request.call_count == 2 + assert isinstance(result, _AsyncResponse) + assert result.status == 200 + + +class TestAsyncHttpClientClose: + """Tests for _AsyncHttpClient.close().""" + + async def test_close_closes_session(self): + """close() closes the session and sets _session to None.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.close = AsyncMock() + client = _AsyncHttpClient(retries=1, session=session) + await client.close() + session.close.assert_called_once() + assert client._session is None + + async def test_close_without_session_is_safe(self): + """close() is safe to call when no session was set.""" + client = _AsyncHttpClient(retries=1) + await client.close() # should not raise + + +class TestAsyncHttpClientLogger: + """Tests for request logging via _HttpLogger integration.""" + + async def test_request_logged_when_logger_set(self): + """Outbound request is logged once when a logger is attached.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + mock_logger.log_request.assert_called_once() + + async def test_response_logged_when_logger_set(self): + """HTTP response is logged when a logger is attached.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + mock_logger.log_response.assert_called_once() + + async def test_error_logged_on_retry(self): + """Transport errors are logged before each retry.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(good_resp), + ] + ) + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=2, backoff=0, session=session, logger=mock_logger) + with patch("asyncio.sleep", new_callable=AsyncMock): + await client._request("get", "https://example.com/data") + mock_logger.log_error.assert_called_once() + + async def test_request_body_logged_from_json_kwarg(self): + """json= kwarg body is extracted and passed to log_request.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("post", "https://example.com/data", json={"key": "value"}) + _, log_kwargs = mock_logger.log_request.call_args + assert log_kwargs["body"] == {"key": "value"} + + async def test_request_body_logged_from_data_kwarg(self): + """data= kwarg body is extracted when json= is absent.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("post", "https://example.com/data", data=b"raw bytes") + _, log_kwargs = mock_logger.log_request.call_args + assert log_kwargs["body"] == b"raw bytes" + + async def test_response_body_decoded_when_body_logging_enabled(self): + """When body_logging_enabled=True, response bytes are decoded and passed to log_response.""" + session = _make_session() + session.request.return_value.__aenter__.return_value.read = AsyncMock(return_value=b'{"ok": true}') + mock_logger = MagicMock() + mock_logger.body_logging_enabled = True + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + _, log_kwargs = mock_logger.log_response.call_args + assert log_kwargs["body"] == '{"ok": true}' + + async def test_response_body_invalid_bytes_replaced_in_logging(self): + """Invalid UTF-8 bytes in response body are replaced (not raised) when body logging is enabled.""" + session = _make_session() + session.request.return_value.__aenter__.return_value.read = AsyncMock(return_value=b"\xff\xfe invalid") + mock_logger = MagicMock() + mock_logger.body_logging_enabled = True + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + _, log_kwargs = mock_logger.log_response.call_args + # errors="replace" means invalid bytes become replacement chars — body is a str, never raises + assert isinstance(log_kwargs["body"], str) diff --git a/tests/unit/aio/data/__init__.py b/tests/unit/aio/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/aio/data/test_async_batch_internal.py b/tests/unit/aio/data/test_async_batch_internal.py new file mode 100644 index 00000000..1a24deed --- /dev/null +++ b/tests/unit/aio/data/test_async_batch_internal.py @@ -0,0 +1,839 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncBatchClient internals.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_batch import _AsyncBatchClient +from PowerPlatform.Dataverse.aio.core._async_http import _AsyncResponse +from PowerPlatform.Dataverse.core.errors import MetadataError, ValidationError +from PowerPlatform.Dataverse.data._batch_base import ( + _RecordCreate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpdate, + _RecordUpsert, + _TableAddColumns, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _TableRemoveColumns, + _QuerySql, + _ChangeSet, + _MAX_BATCH_SIZE, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_batch_client(): + """Return _AsyncBatchClient with a fully-mocked _AsyncODataClient. + + All _build_* methods are pre-mocked so resolver tests can run without + any real OData or HTTP logic. Sync _build_* methods use MagicMock; + async ones use AsyncMock. + """ + od = AsyncMock() + od.api = "https://example.crm.dynamics.com/api/data/v9.2" + od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + od._primary_id_attr = AsyncMock(return_value="accountid") + od._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account", "SchemaName": "Account"} + ) + od._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1", "LogicalName": "new_notes"}) + od._build_create = AsyncMock( + return_value=MagicMock(method="POST", url="https://x/accounts", body="{}", headers=None, content_id=None) + ) + od._build_create_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/CreateMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_update = AsyncMock( + return_value=MagicMock( + method="PATCH", url="https://x/accounts(g)", body="{}", headers={"If-Match": "*"}, content_id=None + ) + ) + od._build_update_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/UpdateMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_delete = AsyncMock( + return_value=MagicMock( + method="DELETE", url="https://x/accounts(g)", body=None, headers={"If-Match": "*"}, content_id=None + ) + ) + od._build_delete_multiple = AsyncMock( + return_value=MagicMock(method="POST", url="https://x/BulkDelete", body="{}", headers=None, content_id=None) + ) + od._build_get = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts(g)", body=None, headers=None, content_id=None) + ) + od._build_upsert = AsyncMock( + return_value=MagicMock(method="PATCH", url="https://x/accounts(k)", body="{}", headers=None, content_id=None) + ) + od._build_upsert_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/UpsertMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_list = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts", body=None, headers=None, content_id=None) + ) + od._build_sql = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts?sql=...", body=None, headers=None, content_id=None) + ) + od._build_delete_entity = MagicMock( + return_value=MagicMock( + method="DELETE", url="https://x/EntityDefinitions(m)", body=None, headers=None, content_id=None + ) + ) + od._build_create_column = MagicMock( + return_value=MagicMock( + method="POST", url="https://x/EntityDefinitions(m)/Attributes", body="{}", headers=None, content_id=None + ) + ) + od._build_delete_column = MagicMock( + return_value=MagicMock( + method="DELETE", + url="https://x/EntityDefinitions(m)/Attributes(a)", + body=None, + headers=None, + content_id=None, + ) + ) + # Sync _build_* for pure-logic table intents inherited from _BatchBase + _raw = lambda method, url: MagicMock(method=method, url=url, body=None, headers=None, content_id=None) + od._build_create_entity = MagicMock(return_value=_raw("POST", "https://x/EntityDefinitions")) + od._build_get_entity = MagicMock(return_value=_raw("GET", "https://x/EntityDefinitions(m)")) + od._build_list_entities = MagicMock(return_value=_raw("GET", "https://x/EntityDefinitions")) + od._build_create_relationship = MagicMock(return_value=_raw("POST", "https://x/RelationshipDefinitions")) + od._build_delete_relationship = MagicMock(return_value=_raw("DELETE", "https://x/RelationshipDefinitions(r)")) + od._build_get_relationship = MagicMock(return_value=_raw("GET", "https://x/RelationshipDefinitions(r)")) + _mock_lookup = MagicMock() + _mock_lookup.to_dict.return_value = {} + _mock_rel = MagicMock() + _mock_rel.to_dict.return_value = {} + od._build_lookup_field_models = MagicMock(return_value=(_mock_lookup, _mock_rel)) + return _AsyncBatchClient(od), od + + +def _batch_resp(status=200, text="", json_payload=None): + """Create a mock _AsyncResponse-compatible response for the batch execute() path.""" + r = MagicMock() + r.status = status + r.status_code = status + r.headers = {"Content-Type": "application/json"} + r.text = text + r.json = MagicMock(return_value=json_payload or {}) + return r + + +# --------------------------------------------------------------------------- +# execute() +# --------------------------------------------------------------------------- + + +class TestExecute: + """Tests for execute(), the public entry point that dispatches the full batch request.""" + + async def test_empty_items_returns_empty_result(self): + """An empty items list short-circuits and returns an empty BatchResult without HTTP.""" + client, _ = _make_batch_client() + result = await client.execute([]) + assert result is not None + + async def test_executes_single_record_create(self): + """A single RecordCreate item causes exactly one _request call.""" + client, od = _make_batch_client() + from PowerPlatform.Dataverse.models.batch import BatchResult + + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()): + await client.execute([item]) + od._request.assert_called_once() + + async def test_executes_with_continue_on_error(self): + """The odata.continue-on-error preference is injected when continue_on_error=True.""" + client, od = _make_batch_client() + from PowerPlatform.Dataverse.models.batch import BatchResult + + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()): + await client.execute([item], continue_on_error=True) + call_kwargs = od._request.call_args.kwargs + headers = call_kwargs.get("headers", {}) + assert "odata.continue-on-error" in headers.get("Prefer", "") + + +# --------------------------------------------------------------------------- +# _resolve_record_create() +# --------------------------------------------------------------------------- + + +class TestResolveRecordCreate: + """Tests for _resolve_record_create() intent-to-request translation.""" + + async def test_single_dict_returns_one_request(self): + """A dict payload produces a single _build_create request.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data={"name": "X"}) + result = await client._resolve_record_create(op) + assert len(result) == 1 + od._build_create.assert_called_once() + + async def test_list_returns_one_create_multiple_request(self): + """A list payload produces a single _build_create_multiple request.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data=[{"name": "X"}, {"name": "Y"}]) + result = await client._resolve_record_create(op) + assert len(result) == 1 + od._build_create_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_update() +# --------------------------------------------------------------------------- + + +class TestResolveRecordUpdate: + """Tests for _resolve_record_update() intent-to-request translation.""" + + async def test_single_id_string_returns_one_patch(self): + """A single string ID with a dict of changes produces one _build_update request.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids="guid-1", changes={"name": "X"}) + result = await client._resolve_record_update(op) + assert len(result) == 1 + od._build_update.assert_called_once() + + async def test_single_id_non_dict_changes_raises(self): + """TypeError is raised when changes is not a dict for a single-ID update.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids="guid-1", changes=["invalid"]) + with pytest.raises(TypeError): + await client._resolve_record_update(op) + + async def test_list_ids_returns_update_multiple(self): + """A list of IDs produces a single _build_update_multiple request.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids=["id-1", "id-2"], changes={"name": "X"}) + result = await client._resolve_record_update(op) + assert len(result) == 1 + od._build_update_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_delete() +# --------------------------------------------------------------------------- + + +class TestResolveRecordDelete: + """Tests for _resolve_record_delete() intent-to-request translation.""" + + async def test_single_id_string(self): + """A single string ID produces one _build_delete request.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids="guid-1") + result = await client._resolve_record_delete(op) + assert len(result) == 1 + od._build_delete.assert_called_once() + + async def test_list_ids_bulk_delete(self): + """A list of IDs with use_bulk_delete=True produces one _build_delete_multiple request.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=True) + result = await client._resolve_record_delete(op) + assert len(result) == 1 + od._build_delete_multiple.assert_called_once() + + async def test_list_ids_sequential_delete(self): + """A list of IDs with use_bulk_delete=False produces one _build_delete per ID.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=False) + result = await client._resolve_record_delete(op) + assert len(result) == 2 + assert od._build_delete.call_count == 2 + + async def test_empty_ids_list_returns_empty(self): + """An empty IDs list produces no requests.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=[]) + result = await client._resolve_record_delete(op) + assert result == [] + + +# --------------------------------------------------------------------------- +# _resolve_record_get() +# --------------------------------------------------------------------------- + + +class TestResolveRecordGet: + """Tests for _resolve_record_get() intent-to-request translation.""" + + async def test_single_get_request(self): + """A RecordGet op produces one _build_get request with the correct arguments.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1", select=["name"]) + result = await client._resolve_record_get(op) + assert len(result) == 1 + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=None + ) + + async def test_passes_expand_to_build_get(self): + """expand= is forwarded from _RecordGet to _build_get.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1", expand=["primarycontactid"]) + await client._resolve_record_get(op) + od._build_get.assert_called_once_with( + "account", "guid-1", select=None, expand=["primarycontactid"], include_annotations=None + ) + + async def test_passes_include_annotations_to_build_get(self): + """include_annotations= is forwarded from _RecordGet to _build_get.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet(table="account", record_id="guid-1", include_annotations=annotation) + await client._resolve_record_get(op) + od._build_get.assert_called_once_with( + "account", "guid-1", select=None, expand=None, include_annotations=annotation + ) + + async def test_passes_all_params_to_build_get(self): + """All _RecordGet fields are forwarded together to _build_get.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet( + table="account", + record_id="guid-1", + select=["name"], + expand=["primarycontactid"], + include_annotations=annotation, + ) + result = await client._resolve_record_get(op) + assert len(result) == 1 + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=["primarycontactid"], include_annotations=annotation + ) + + +# --------------------------------------------------------------------------- +# _resolve_record_list() +# --------------------------------------------------------------------------- + + +class TestResolveRecordList: + """Tests for _resolve_record_list() intent-to-request translation.""" + + async def test_produces_one_request(self): + """A _RecordList op produces exactly one _build_list request.""" + client, od = _make_batch_client() + op = _RecordList(table="account") + result = await client._resolve_record_list(op) + assert len(result) == 1 + od._build_list.assert_called_once() + + async def test_passes_table_to_build_list(self): + """The table name is forwarded to _build_list as first positional arg.""" + client, od = _make_batch_client() + op = _RecordList(table="contact") + await client._resolve_record_list(op) + call_args = od._build_list.call_args + assert call_args[0][0] == "contact" + + async def test_passes_filter(self): + """filter= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", filter="statecode eq 0") + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["filter"] == "statecode eq 0" + + async def test_passes_select(self): + """select= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", select=["name", "revenue"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["select"] == ["name", "revenue"] + + async def test_passes_top(self): + """top= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", top=50) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["top"] == 50 + + async def test_passes_orderby(self): + """orderby= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", orderby=["name asc"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["orderby"] == ["name asc"] + + async def test_passes_expand(self): + """expand= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", expand=["primarycontactid"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["expand"] == ["primarycontactid"] + + async def test_passes_page_size(self): + """page_size= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", page_size=200) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["page_size"] == 200 + + async def test_passes_count(self): + """count=True is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", count=True) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["count"] is True + + async def test_passes_include_annotations(self): + """include_annotations= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordList(table="account", include_annotations=annotation) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["include_annotations"] == annotation + + async def test_resolve_item_dispatch(self): + """_resolve_item dispatches _RecordList correctly.""" + client, od = _make_batch_client() + op = _RecordList(table="account", filter="statecode eq 0") + result = await client._resolve_item(op) + assert len(result) == 1 + od._build_list.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_upsert() +# --------------------------------------------------------------------------- + + +class TestResolveRecordUpsert: + """Tests for _resolve_record_upsert() intent-to-request translation.""" + + async def test_single_item_calls_build_upsert(self): + """A single UpsertItem produces one _build_upsert request.""" + client, od = _make_batch_client() + item = UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "X"}) + op = _RecordUpsert(table="account", items=[item]) + result = await client._resolve_record_upsert(op) + assert len(result) == 1 + od._build_upsert.assert_called_once() + + async def test_multiple_items_calls_build_upsert_multiple(self): + """Multiple UpsertItems produce a single _build_upsert_multiple request.""" + client, od = _make_batch_client() + items = [ + UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "X"}), + UpsertItem(alternate_key={"accountnumber": "B"}, record={"name": "Y"}), + ] + op = _RecordUpsert(table="account", items=items) + result = await client._resolve_record_upsert(op) + assert len(result) == 1 + od._build_upsert_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_table_delete() +# --------------------------------------------------------------------------- + + +class TestResolveTableDelete: + """Tests for _resolve_table_delete() intent-to-request translation.""" + + async def test_resolves_to_delete_request(self): + """A TableDelete op resolves to a _build_delete_entity call using the table's MetadataId.""" + client, od = _make_batch_client() + op = _TableDelete(table="account") + result = await client._resolve_table_delete(op) + assert len(result) == 1 + od._build_delete_entity.assert_called_once_with("meta-1") + + async def test_table_not_found_raises(self): + """MetadataError is raised when the target table does not exist in metadata.""" + client, od = _make_batch_client() + od._get_entity_by_table_schema_name = AsyncMock(return_value=None) + op = _TableDelete(table="nonexistent") + with pytest.raises(MetadataError): + await client._resolve_table_delete(op) + + +# --------------------------------------------------------------------------- +# _resolve_table_add_columns() +# --------------------------------------------------------------------------- + + +class TestResolveTableAddColumns: + """Tests for _resolve_table_add_columns() intent-to-request translation.""" + + async def test_resolves_to_create_column_requests(self): + """Each column in the op produces one _build_create_column request.""" + client, od = _make_batch_client() + op = _TableAddColumns(table="account", columns={"col1": "string", "col2": "decimal"}) + result = await client._resolve_table_add_columns(op) + assert len(result) == 2 + assert od._build_create_column.call_count == 2 + + +# --------------------------------------------------------------------------- +# _resolve_table_remove_columns() +# --------------------------------------------------------------------------- + + +class TestResolveTableRemoveColumns: + """Tests for _resolve_table_remove_columns() intent-to-request translation.""" + + async def test_resolves_to_delete_column_requests(self): + """Each column in the list produces one _build_delete_column request.""" + client, od = _make_batch_client() + op = _TableRemoveColumns(table="account", columns=["col1", "col2"]) + result = await client._resolve_table_remove_columns(op) + assert len(result) == 2 + + async def test_string_column_name(self): + """A single column name supplied as a string produces one delete request.""" + client, od = _make_batch_client() + op = _TableRemoveColumns(table="account", columns="col1") + result = await client._resolve_table_remove_columns(op) + assert len(result) == 1 + + async def test_column_not_found_raises(self): + """MetadataError is raised when attribute metadata returns None for the column.""" + client, od = _make_batch_client() + od._get_attribute_metadata = AsyncMock(return_value=None) + op = _TableRemoveColumns(table="account", columns=["nonexistent"]) + with pytest.raises(MetadataError): + await client._resolve_table_remove_columns(op) + + async def test_attr_missing_metadata_id_raises(self): + """MetadataError is raised when attribute metadata lacks a MetadataId field.""" + client, od = _make_batch_client() + od._get_attribute_metadata = AsyncMock(return_value={"LogicalName": "col1"}) + op = _TableRemoveColumns(table="account", columns=["col1"]) + with pytest.raises(MetadataError): + await client._resolve_table_remove_columns(op) + + +# --------------------------------------------------------------------------- +# _resolve_query_sql() +# --------------------------------------------------------------------------- + + +class TestResolveQuerySql: + """Tests for _resolve_query_sql() intent-to-request translation.""" + + async def test_resolves_to_get_request(self): + """A QuerySql op produces one _build_sql request with the SQL statement.""" + client, od = _make_batch_client() + op = _QuerySql(sql="SELECT name FROM account") + result = await client._resolve_query_sql(op) + assert len(result) == 1 + od._build_sql.assert_called_once_with("SELECT name FROM account") + + +# --------------------------------------------------------------------------- +# _resolve_one() — changeset item must produce exactly 1 request +# --------------------------------------------------------------------------- + + +class TestResolveOne: + """Tests for _resolve_one(), which enforces the single-request contract for changeset items.""" + + async def test_single_request_returned(self): + """An op that resolves to exactly one request is returned without error.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1") + req = await client._resolve_one(op) + assert req is not None + + async def test_multi_request_item_raises(self): + """ValidationError is raised when an op resolves to more than one request.""" + client, od = _make_batch_client() + # _RecordDelete with a list produces multiple requests (one per ID) + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=False) + with pytest.raises(ValidationError, match="exactly one"): + await client._resolve_one(op) + + +# --------------------------------------------------------------------------- +# _resolve_all() — changeset handling +# --------------------------------------------------------------------------- + + +class TestResolveAll: + """Tests for _resolve_all(), which dispatches items and wraps changeset ops.""" + + async def test_empty_changeset_skipped(self): + """A ChangeSet with no operations is silently skipped without error.""" + client, od = _make_batch_client() + cs = _ChangeSet(operations=[]) + result = await client._resolve_all([cs]) + assert result == [] + + async def test_changeset_with_operations(self): + """A ChangeSet with one operation produces one _ChangeSetBatchItem in the result.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data={"name": "X"}) + cs = _ChangeSet(operations=[op]) + result = await client._resolve_all([cs]) + assert len(result) == 1 + + async def test_unknown_item_type_raises(self): + """ValidationError is raised when an unrecognised item type is passed to _resolve_item.""" + client, od = _make_batch_client() + with pytest.raises(ValidationError, match="Unknown batch item type"): + await client._resolve_item("not-a-valid-type") + + +# --------------------------------------------------------------------------- +# _require_entity_metadata() +# --------------------------------------------------------------------------- + + +class TestRequireEntityMetadata: + """Tests for _require_entity_metadata(), which resolves a table's MetadataId or raises.""" + + async def test_returns_metadata_id(self): + """The MetadataId string is returned when the table exists in entity metadata.""" + client, od = _make_batch_client() + meta_id = await client._require_entity_metadata("account") + assert meta_id == "meta-1" + + async def test_not_found_raises(self): + """MetadataError is raised when the API returns no entity definition for the table.""" + client, od = _make_batch_client() + od._get_entity_by_table_schema_name = AsyncMock(return_value=None) + with pytest.raises(MetadataError): + await client._require_entity_metadata("nonexistent") + + +# --------------------------------------------------------------------------- +# _AsyncResponse +# --------------------------------------------------------------------------- + + +class TestHttpResponse: + """Tests for _AsyncResponse — the materialized response returned by _AsyncHttpClient.""" + + def test_json_parses_body(self): + """json() parses the body bytes as JSON.""" + r = _AsyncResponse(200, {"Content-Type": "application/json"}, b'{"value": []}') + assert r.json() == {"value": []} + + def test_json_empty_body_returns_empty_dict(self): + """json() returns {} for an empty body.""" + r = _AsyncResponse(200, {}, b"") + assert r.json() == {} + + def test_status_and_status_code_equal(self): + """status and status_code are both set and equal.""" + r = _AsyncResponse(207, {}, b"") + assert r.status == 207 + assert r.status_code == 207 + + def test_text_decodes_body(self): + """text property decodes body bytes as UTF-8.""" + r = _AsyncResponse(200, {}, b"body text") + assert r.text == "body text" + + def test_text_empty_body(self): + """text property returns empty string for empty body.""" + r = _AsyncResponse(200, {}, b"") + assert r.text == "" + + +# --------------------------------------------------------------------------- +# execute() edge cases +# --------------------------------------------------------------------------- + + +class TestExecuteEdgeCases: + """Tests for execute() error paths not covered by TestExecute.""" + + async def test_batch_size_exceeded_raises(self): + """execute() raises ValidationError when more than _MAX_BATCH_SIZE items are resolved.""" + client, od = _make_batch_client() + # _RecordCreate × (_MAX_BATCH_SIZE + 1) — each resolves to one request + items = [_RecordCreate(table="account", data={"name": f"X{i}"}) for i in range(_MAX_BATCH_SIZE + 1)] + with pytest.raises(ValidationError, match="exceeds the limit"): + await client.execute(items) + + async def test_response_passed_directly_to_parse_batch_response(self): + """execute() passes the HTTP response object directly to _parse_batch_response.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + client, od = _make_batch_client() + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()) as mock_parse: + await client.execute([item]) + mock_parse.assert_called_once_with(resp_mock) + + +# --------------------------------------------------------------------------- +# _resolve_all() edge cases +# --------------------------------------------------------------------------- + + +class TestResolveAllEdgeCases: + """Tests for _resolve_all() paths not covered elsewhere.""" + + async def test_empty_changeset_is_skipped(self): + """An empty _ChangeSet is silently dropped from the resolved list.""" + client, od = _make_batch_client() + cs = _ChangeSet(_counter=[1]) + # No operations added — operations list is empty + result = await client._resolve_all([cs]) + assert result == [] + + async def test_non_changeset_item_extended(self): + """Non-changeset items are resolved and extended into the flat result.""" + client, od = _make_batch_client() + item = _RecordCreate(table="account", data={"name": "X"}) + result = await client._resolve_all([item]) + assert len(result) == 1 + + +# --------------------------------------------------------------------------- +# _resolve_item() full dispatch coverage +# --------------------------------------------------------------------------- + + +class TestResolveItemDispatch: + """One test per intent type — drives every branch of _resolve_item(). + + Each test replaces the specific resolver method with a mock so only the + dispatch logic is exercised. Record/SQL resolvers are async (awaited); + pure table resolvers inherited from _BatchBase are sync (not awaited). + """ + + _sentinel = MagicMock(method="GET", url="https://x/test", body=None, headers=None, content_id=None) + + def _async_mock(self): + return AsyncMock(return_value=[self._sentinel]) + + def _sync_mock(self): + return MagicMock(return_value=[self._sentinel]) + + async def test_dispatch_record_update(self): + client, _ = _make_batch_client() + client._resolve_record_update = self._async_mock() + result = await client._resolve_item(_RecordUpdate(table="account", ids="g", changes={"name": "X"})) + client._resolve_record_update.assert_called_once() + + async def test_dispatch_record_upsert(self): + client, _ = _make_batch_client() + client._resolve_record_upsert = self._async_mock() + item = UpsertItem(alternate_key={"k": "v"}, record={"name": "X"}) + result = await client._resolve_item(_RecordUpsert(table="account", items=[item])) + client._resolve_record_upsert.assert_called_once() + + async def test_dispatch_table_create(self): + client, _ = _make_batch_client() + client._resolve_table_create = self._sync_mock() + result = await client._resolve_item(_TableCreate(table="new_Test", columns={"new_Name": "string"})) + client._resolve_table_create.assert_called_once() + + async def test_dispatch_table_delete(self): + client, _ = _make_batch_client() + client._resolve_table_delete = self._async_mock() + result = await client._resolve_item(_TableDelete(table="new_Test")) + client._resolve_table_delete.assert_called_once() + + async def test_dispatch_table_get(self): + client, _ = _make_batch_client() + client._resolve_table_get = self._sync_mock() + result = await client._resolve_item(_TableGet(table="account")) + client._resolve_table_get.assert_called_once() + + async def test_dispatch_table_list(self): + client, _ = _make_batch_client() + client._resolve_table_list = self._sync_mock() + result = await client._resolve_item(_TableList()) + client._resolve_table_list.assert_called_once() + + async def test_dispatch_table_add_columns(self): + client, _ = _make_batch_client() + client._resolve_table_add_columns = self._async_mock() + result = await client._resolve_item(_TableAddColumns(table="account", columns={"new_X": "string"})) + client._resolve_table_add_columns.assert_called_once() + + async def test_dispatch_table_remove_columns(self): + client, _ = _make_batch_client() + client._resolve_table_remove_columns = self._async_mock() + result = await client._resolve_item(_TableRemoveColumns(table="account", columns="new_X")) + client._resolve_table_remove_columns.assert_called_once() + + async def test_dispatch_table_create_one_to_many(self): + client, _ = _make_batch_client() + client._resolve_table_create_one_to_many = self._sync_mock() + op = _TableCreateOneToMany(relationship=MagicMock(), lookup=MagicMock()) + result = await client._resolve_item(op) + client._resolve_table_create_one_to_many.assert_called_once() + + async def test_dispatch_table_create_many_to_many(self): + client, _ = _make_batch_client() + client._resolve_table_create_many_to_many = self._sync_mock() + op = _TableCreateManyToMany(relationship=MagicMock()) + result = await client._resolve_item(op) + client._resolve_table_create_many_to_many.assert_called_once() + + async def test_dispatch_table_delete_relationship(self): + client, _ = _make_batch_client() + client._resolve_table_delete_relationship = self._sync_mock() + result = await client._resolve_item(_TableDeleteRelationship(relationship_id="rel-guid")) + client._resolve_table_delete_relationship.assert_called_once() + + async def test_dispatch_table_get_relationship(self): + client, _ = _make_batch_client() + client._resolve_table_get_relationship = self._sync_mock() + result = await client._resolve_item(_TableGetRelationship(schema_name="new_a_c")) + client._resolve_table_get_relationship.assert_called_once() + + async def test_dispatch_table_create_lookup_field(self): + client, _ = _make_batch_client() + client._resolve_table_create_lookup_field = self._sync_mock() + result = await client._resolve_item( + _TableCreateLookupField( + referencing_table="contact", + lookup_field_name="new_accountid", + referenced_table="account", + ) + ) + client._resolve_table_create_lookup_field.assert_called_once() + + async def test_dispatch_query_sql(self): + client, _ = _make_batch_client() + client._resolve_query_sql = self._async_mock() + result = await client._resolve_item(_QuerySql(sql="SELECT accountid FROM account")) + client._resolve_query_sql.assert_called_once() + + async def test_dispatch_unknown_type_raises(self): + """An unrecognised intent type raises ValidationError.""" + client, _ = _make_batch_client() + with pytest.raises(ValidationError, match="Unknown batch item type"): + await client._resolve_item("not-an-intent") diff --git a/tests/unit/aio/data/test_async_odata_internal.py b/tests/unit/aio/data/test_async_odata_internal.py new file mode 100644 index 00000000..a064c1ed --- /dev/null +++ b/tests/unit/aio/data/test_async_odata_internal.py @@ -0,0 +1,1846 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncODataClient internals (mocking _request at the HTTP boundary).""" + +import json +import time +import warnings +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError, ValidationError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked out at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(json_data=None, status=200, headers=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.text = json.dumps(json_data) if json_data is not None else "" + r.json = MagicMock(return_value=json_data if json_data is not None else {}) + return r + + +def _entity_def( + entity_set="accounts", + pk="accountid", + meta_id="meta-001", + schema="Account", + logical="account", +): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": entity_set, + "PrimaryIdAttribute": pk, + "MetadataId": meta_id, + "SchemaName": schema, + } + ] + } + + +def _seed_cache(client: _AsyncODataClient, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +# --------------------------------------------------------------------------- +# close() +# --------------------------------------------------------------------------- + + +class TestClose: + """Tests for the close() lifecycle method.""" + + async def test_close_delegates_to_http(self): + """close() forwards to the underlying HTTP client's close() exactly once.""" + client = _make_client() + client._http.close = AsyncMock() + await client.close() + client._http.close.assert_called_once() + + async def test_close_clears_entity_set_cache(self): + """close() empties the entity-set lookup cache so stale entries don't persist.""" + client = _make_client() + _seed_cache(client) + client._http.close = AsyncMock() + await client.close() + assert len(client._logical_to_entityset_cache) == 0 + + +# --------------------------------------------------------------------------- +# _request() — tests actual implementation via _raw_request mock +# --------------------------------------------------------------------------- + + +class TestRequest: + """Tests for _request() error extraction. + + These tests mock _raw_request (one level below _request) so the real + header-building, status-checking, and error-parsing code runs. + """ + + def _auth_client(self): + """Return a client with a real auth mock but _raw_request not yet patched.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + return _AsyncODataClient(auth, "https://example.crm.dynamics.com") + + async def test_ok_response_returned(self): + """2xx responses are returned to the caller without raising.""" + client = self._auth_client() + client._raw_request = AsyncMock(return_value=_resp(status=200, json_data={"value": []})) + r = await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert r.status == 200 + + async def test_error_with_nested_error_object(self): + """Nested error.code / error.message body structure is parsed into HttpError.""" + client = self._auth_client() + body = {"error": {"code": "0x80040265", "message": "Not found"}} + client._raw_request = AsyncMock(return_value=_resp(status=404, json_data=body)) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 404 + assert "Not found" in str(exc.value) + + async def test_error_with_message_at_root(self): + """A top-level message key in the body is used when error nesting is absent.""" + client = self._auth_client() + body = {"message": "Root-level message"} + client._raw_request = AsyncMock(return_value=_resp(status=400, json_data=body)) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert "Root-level message" in str(exc.value) + + async def test_error_non_json_body_handled(self): + """Non-JSON response body falls back to HTTP status code as the error message.""" + client = self._auth_client() + r = MagicMock() + r.status = 503 + r.headers = {} + r.text = "Service Unavailable" + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 503 + + async def test_retry_after_header_parsed(self): + """Retry-After header value is stored as an integer in the error's details dict.""" + client = self._auth_client() + body = {"error": {"code": "429", "message": "Too many requests"}} + r = _resp(status=429, json_data=body, headers={"Retry-After": "60"}) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("retry_after") == 60 + + async def test_service_request_id_extracted(self): + """x-ms-service-request-id header is stored in the error's details dict.""" + client = self._auth_client() + r = _resp( + status=500, + json_data={"error": {"code": "err", "message": "fail"}}, + headers={"x-ms-service-request-id": "srv-req-1"}, + ) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("service_request_id") == "srv-req-1" + + +# --------------------------------------------------------------------------- +# _create() +# --------------------------------------------------------------------------- + + +class TestCreate: + """Tests for _create() single-record creation.""" + + async def test_returns_guid_from_odata_entity_id(self): + """GUID is extracted from the OData-EntityId response header.""" + client = _make_client() + _seed_cache(client) + guid = "12345678-1234-1234-1234-123456789012" + client._request.return_value = _resp( + status=204, + headers={"OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/accounts({guid})"}, + ) + result = await client._create("accounts", "account", {"amount": 100}) + assert result == guid + + async def test_returns_guid_from_location_header(self): + """Location header is used as fallback when OData-EntityId is absent.""" + client = _make_client() + _seed_cache(client) + guid = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + client._request.return_value = _resp( + status=204, + headers={"Location": f"https://example.crm.dynamics.com/api/data/v9.2/accounts({guid})"}, + ) + result = await client._create("accounts", "account", {"amount": 100}) + assert result == guid + + async def test_raises_when_no_guid_in_headers(self): + """RuntimeError is raised when neither OData-EntityId nor Location contains a GUID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204, headers={}) + with pytest.raises(RuntimeError, match="GUID"): + await client._create("accounts", "account", {"amount": 100}) + + +# --------------------------------------------------------------------------- +# _create_multiple() +# --------------------------------------------------------------------------- + + +class TestCreateMultiple: + """Tests for _create_multiple() bulk record creation.""" + + async def test_returns_ids_from_ids_key(self): + """IDs are extracted from the top-level Ids key in the response body.""" + client = _make_client() + _seed_cache(client) + guids = ["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"] + client._request.return_value = _resp(json_data={"Ids": guids}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}, {"amount": 2}]) + assert result == guids + + async def test_returns_ids_from_value_list(self): + """IDs are extracted from value list entries when the Ids key is absent.""" + client = _make_client() + _seed_cache(client) + guid = "11111111-1111-1111-1111-111111111111" + client._request.return_value = _resp(json_data={"value": [{"accountid": guid}]}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [guid] + + async def test_empty_body_returns_empty_list(self): + """An empty response body returns an empty list without raising.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + async def test_non_dict_records_raises(self): + """TypeError is raised when the records list contains non-dict items.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError): + await client._create_multiple("accounts", "account", ["not-a-dict"]) + + +# --------------------------------------------------------------------------- +# _update() / _update_by_ids() / _update_multiple() +# --------------------------------------------------------------------------- + + +class TestUpdate: + """Tests for _update(), _update_by_ids(), and _update_multiple().""" + + async def test_update_patches_record(self): + """_update() issues a PATCH request for the given record ID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update("account", "guid-1", {"telephone1": "555"}) + assert client._request.called + + async def test_update_by_ids_broadcast_dict(self): + """A dict for changes is broadcast to all IDs via UpdateMultiple.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update_by_ids("account", ["id-1", "id-2"], {"statecode": 0}) + assert client._request.called + + async def test_update_by_ids_paired_list(self): + """A list for changes is applied pairwise with the corresponding IDs.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update_by_ids("account", ["id-1"], [{"name": "A"}]) + assert client._request.called + + async def test_update_by_ids_empty_list_is_noop(self): + """An empty ID list short-circuits without issuing any HTTP request.""" + client = _make_client() + result = await client._update_by_ids("account", [], {"statecode": 0}) + assert result is None + client._request.assert_not_called() + + async def test_update_by_ids_mismatched_length_raises(self): + """ValueError is raised when the changes list is shorter than the ID list.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=200) + with pytest.raises(ValueError): + await client._update_by_ids("account", ["id-1", "id-2"], [{"name": "A"}]) + + async def test_update_by_ids_invalid_changes_type_raises(self): + """TypeError is raised when changes is neither a dict nor a list.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError): + await client._update_by_ids("account", ["id-1"], "invalid") + + async def test_update_multiple_empty_records_raises(self): + """TypeError is raised when the records list is empty.""" + client = _make_client() + with pytest.raises(TypeError): + await client._update_multiple("accounts", "account", []) + + async def test_update_multiple_non_list_raises(self): + """TypeError is raised when the records argument is not a list.""" + client = _make_client() + with pytest.raises(TypeError): + await client._update_multiple("accounts", "account", "not-a-list") + + +# --------------------------------------------------------------------------- +# _delete() / _delete_multiple() +# --------------------------------------------------------------------------- + + +class TestDelete: + """Tests for _delete() single-record deletion and _delete_multiple() bulk deletion.""" + + async def test_delete_calls_request(self): + """_delete() issues a DELETE request for the given record ID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._delete("account", "guid-1") + assert client._request.called + + async def test_delete_multiple_returns_job_id(self): + """JobId from the async BulkDelete response is returned to the caller.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"JobId": "job-guid-1"}, status=202) + result = await client._delete_multiple("account", ["id-1", "id-2"]) + assert result == "job-guid-1" + + async def test_delete_multiple_empty_ids_returns_none(self): + """An empty ID list short-circuits without issuing any HTTP request.""" + client = _make_client() + result = await client._delete_multiple("account", []) + assert result is None + client._request.assert_not_called() + + async def test_delete_multiple_no_job_id_in_body(self): + """None is returned when the response body does not contain a JobId key.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={}, status=204) + result = await client._delete_multiple("account", ["id-1"]) + assert result is None + + +# --------------------------------------------------------------------------- +# _get() / _get_multiple() +# --------------------------------------------------------------------------- + + +class TestGet: + """Tests for _get() single-record fetch.""" + + async def test_get_returns_record(self): + """The full record dict from the response body is returned unchanged.""" + client = _make_client() + _seed_cache(client) + record = {"accountid": "guid-1", "name": "Contoso"} + client._request.return_value = _resp(json_data=record, status=200) + result = await client._get("account", "guid-1") + assert result == record + + async def test_get_with_select_param(self): + """The select list is forwarded as a $select query parameter.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"name": "Contoso"}, status=200) + result = await client._get("account", "guid-1", select=["name"]) + assert result == {"name": "Contoso"} + + +class TestGetMultiple: + """Tests for _get_multiple() async generator for paged results.""" + + async def test_single_page_yielded(self): + """A single-page response produces exactly one batch from the generator.""" + client = _make_client() + _seed_cache(client) + page = {"value": [{"accountid": "1"}, {"accountid": "2"}]} + client._request.return_value = _resp(json_data=page, status=200) + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 1 + assert len(pages[0]) == 2 + + async def test_follows_next_link(self): + """@odata.nextLink is followed to fetch subsequent pages automatically.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=xyz" + page1 = {"value": [{"accountid": "1"}], "@odata.nextLink": next_url} + page2 = {"value": [{"accountid": "2"}]} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 2 + + async def test_empty_value_not_yielded(self): + """A page with an empty value list produces no output from the generator.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"value": []}, status=200) + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 0 + + async def test_with_all_params(self): + """All optional query parameters are forwarded in the outbound request.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"value": []}, status=200) + async for _ in client._get_multiple( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=10, + expand=["primarycontactid"], + page_size=5, + count=True, + include_annotations="*", + ): + pass + call = client._request.call_args + assert call is not None + kwargs = call.kwargs + assert "headers" in kwargs or kwargs.get("params") is not None + + +# --------------------------------------------------------------------------- +# _query_sql() +# --------------------------------------------------------------------------- + + +class TestQuerySql: + """Tests for _query_sql() which executes Dataverse SQL against the TDS endpoint.""" + + async def test_raises_if_not_string(self): + """ValidationError is raised when the SQL argument is not a string.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._query_sql(123) + + async def test_raises_if_empty(self): + """ValidationError is raised for a blank or whitespace-only SQL string.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._query_sql(" ") + + async def test_returns_rows_from_value(self): + """Rows are extracted from the value list in a standard OData response body.""" + client = _make_client() + _seed_cache(client) + rows = [{"name": "A"}, {"name": "B"}] + client._request.return_value = _resp(json_data={"value": rows}, status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == rows + + async def test_returns_list_body_directly(self): + """A list response body (rather than {value: [...]}) is accepted as rows directly.""" + client = _make_client() + _seed_cache(client) + rows = [{"name": "A"}] + client._request.return_value = _resp(json_data=rows, status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == rows + + async def test_follows_next_link(self): + """Pagination via @odata.nextLink concatenates all rows across pages.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name+FROM+account&page=2" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": next_url} + page2 = {"value": [{"name": "B"}]} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + result = await client._query_sql("SELECT name FROM account") + assert len(result) == 2 + + async def test_warns_and_stops_on_url_cycle(self): + """A repeated nextLink triggers a warning and stops pagination to prevent an infinite loop.""" + client = _make_client() + _seed_cache(client) + cycle_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name+FROM+account&page=1" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": cycle_url} + page2 = {"value": [{"name": "B"}], "@odata.nextLink": cycle_url} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + # Cycle detected after page 2's nextLink repeats; pages 1 and 2 are collected. + assert any("same nextLink" in str(w.message) for w in caught) + assert len(result) == 2 + + async def test_stops_on_non_dict_page_body(self): + """A non-dict page body halts pagination and discards the malformed page.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name&page=2" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": next_url} + # page2 is a list, not a dict — break condition + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=["not-a-dict"])] + result = await client._query_sql("SELECT name FROM account") + assert len(result) == 1 + + +# --------------------------------------------------------------------------- +# _entity_set_from_schema_name() +# --------------------------------------------------------------------------- + + +class TestEntitySetResolution: + """Tests for _entity_set_from_schema_name() cache lookup and HTTP fetch.""" + + async def test_cache_hit_skips_http(self): + """A pre-populated cache entry is returned without any HTTP call.""" + client = _make_client() + _seed_cache(client) + result = await client._entity_set_from_schema_name("account") + client._request.assert_not_called() + assert result == "accounts" + + async def test_fetches_and_caches(self): + """On a cache miss, the entity set is fetched from the API and cached for reuse.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + result = await client._entity_set_from_schema_name("account") + assert result == "accounts" + # Subsequent call must hit the cache, not the API. + client._request.reset_mock() + result2 = await client._entity_set_from_schema_name("account") + client._request.assert_not_called() + assert result2 == "accounts" + + async def test_caches_primary_id_attr(self): + """The primary ID attribute is cached alongside the entity set name.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(pk="accountid"), status=200) + await client._entity_set_from_schema_name("account") + key = client._normalize_cache_key("account") + assert client._logical_primaryid_cache.get(key) == "accountid" + + async def test_not_found_raises_metadata_error(self): + """MetadataError is raised when the API returns an empty value list.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="Unable to resolve"): + await client._entity_set_from_schema_name("nonexistent") + + async def test_plural_name_includes_hint(self): + """The error message hints at a plural-name mistake when the input ends with 's'.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="plural"): + await client._entity_set_from_schema_name("accounts") + + async def test_missing_entity_set_name_raises(self): + """MetadataError is raised when the entity definition lacks an EntitySetName.""" + client = _make_client() + client._request.return_value = _resp( + json_data={"value": [{"LogicalName": "account", "MetadataId": "m1"}]}, + status=200, + ) + with pytest.raises(MetadataError, match="EntitySetName"): + await client._entity_set_from_schema_name("account") + + async def test_empty_name_raises_value_error(self): + """ValueError is raised immediately for an empty table schema name.""" + client = _make_client() + with pytest.raises(ValueError): + await client._entity_set_from_schema_name("") + + +# --------------------------------------------------------------------------- +# _get_table_info() / _list_tables() / _delete_table() +# --------------------------------------------------------------------------- + + +class TestTableInfo: + """Tests for _get_table_info() entity-definition summary lookup.""" + + async def test_get_table_info_found(self): + """A found table returns a dict containing entity_set_name and columns_created.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + result = await client._get_table_info("account") + assert result is not None + assert result["entity_set_name"] == "accounts" + assert result["columns_created"] == [] + + async def test_get_table_info_not_found(self): + """None is returned when the table does not exist in metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + result = await client._get_table_info("nonexistent") + assert result is None + + +class TestListTables: + """Tests for _list_tables() entity-definition list retrieval.""" + + async def test_list_tables_returns_value(self): + """The value list from the EntityDefinitions response is returned unchanged.""" + client = _make_client() + tables = [{"LogicalName": "account"}] + client._request.return_value = _resp(json_data={"value": tables}, status=200) + result = await client._list_tables() + assert result == tables + + async def test_list_tables_with_filter_and_select(self): + """Optional filter and select parameters are forwarded to the API request.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + result = await client._list_tables(filter="IsPrivate eq false", select=["LogicalName"]) + assert result == [] + + +class TestDeleteTable: + """Tests for _delete_table() metadata-level table removal.""" + + async def test_delete_calls_delete_request(self): + """Two requests are issued: one to resolve the MetadataId, one DELETE.""" + client = _make_client() + client._request.side_effect = [_resp(json_data=_entity_def()), _resp(status=204)] + await client._delete_table("account") + assert client._request.call_count == 2 + + async def test_delete_not_found_raises(self): + """MetadataError is raised when the target table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._delete_table("nonexistent") + + +# --------------------------------------------------------------------------- +# _create_table() +# --------------------------------------------------------------------------- + + +class TestCreateTable: + """Tests for _create_table() custom table provisioning.""" + + async def test_table_already_exists_raises(self): + """MetadataError is raised when a table with the same schema name already exists.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + with pytest.raises(MetadataError, match="already exists"): + await client._create_table("account", {}) + + async def test_success_with_columns(self): + """Table and typed columns are created; the returned dict lists columns_created.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp( + json_data=_entity_def(entity_set="new_products", schema="new_Product", logical="new_product"), + status=200, + ) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {"new_Price": "decimal"}) + assert result["table_schema_name"] == "new_Product" + assert "new_Price" in result["columns_created"] + + async def test_success_with_primary_column(self): + """An explicit primary_column_schema_name is accepted without error.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp(json_data=_entity_def(entity_set="new_products"), status=200) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {}, primary_column_schema_name="new_ProductName") + assert result is not None + + async def test_success_with_display_name(self): + """A string display_name is accepted and forwarded to the API.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp(json_data=_entity_def(entity_set="new_products"), status=200) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {}, display_name="Product") + assert result is not None + + async def test_unsupported_column_type_raises(self): + """ValueError is raised before the POST when a column type string is unrecognised.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(ValueError, match="Unsupported"): + await client._create_table("new_Product", {"col": "badtype"}) + + async def test_empty_solution_name_raises(self): + """ValueError is raised when solution_unique_name is an empty string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(ValueError, match="cannot be empty"): + await client._create_table("new_Product", {}, solution_unique_name="") + + async def test_non_string_solution_raises(self): + """TypeError is raised when solution_unique_name is not a string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(TypeError): + await client._create_table("new_Product", {}, solution_unique_name=42) + + async def test_invalid_display_name_raises(self): + """TypeError is raised when display_name is not a string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(TypeError): + await client._create_table("new_Product", {}, display_name=123) + + +# --------------------------------------------------------------------------- +# _create_columns() / _delete_columns() +# --------------------------------------------------------------------------- + + +class TestCreateColumns: + """Tests for _create_columns() column provisioning on an existing table.""" + + async def test_creates_string_column(self): + """A string-typed column is created and its name returned in the result list.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp] + result = await client._create_columns("account", {"new_Notes": "string"}) + assert result == ["new_Notes"] + + async def test_empty_columns_dict_raises(self): + """TypeError is raised when the columns dict is empty.""" + client = _make_client() + with pytest.raises(TypeError, match="non-empty dict"): + await client._create_columns("account", {}) + + async def test_non_dict_columns_raises(self): + """TypeError is raised when the columns argument is not a dict.""" + client = _make_client() + with pytest.raises(TypeError): + await client._create_columns("account", ["col"]) + + async def test_table_not_found_raises(self): + """MetadataError is raised when the parent table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._create_columns("nonexistent", {"col": "string"}) + + async def test_unsupported_type_raises_validation_error(self): + """ValidationError is raised for an unrecognised column type string.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + client._request.return_value = entity_resp + with pytest.raises(ValidationError): + await client._create_columns("account", {"col": "badtype"}) + + async def test_optionset_column_flushes_cache(self): + """Creating a column whose payload includes OptionSet invalidates the picklist label cache. + + Boolean columns produce an OptionSet payload, which triggers the same cache-flush + path used by choice/picklist columns. + """ + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp] + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {"old": {}}} + result = await client._create_columns("account", {"new_Status": "bool"}) + assert result == ["new_Status"] + assert key not in client._picklist_label_cache + + +class TestDeleteColumns: + """Tests for _delete_columns() column removal from an existing table.""" + + async def test_string_column_name(self): + """A single column name supplied as a string is accepted and deleted.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_notes"}]}, status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + result = await client._delete_columns("account", "new_Notes") + assert result == ["new_Notes"] + + async def test_list_column_names(self): + """Column names supplied as a list are each deleted in turn.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_notes"}]}, status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + result = await client._delete_columns("account", ["new_Notes"]) + assert result == ["new_Notes"] + + async def test_invalid_type_raises(self): + """TypeError is raised when the columns argument is neither str nor list.""" + client = _make_client() + with pytest.raises(TypeError): + await client._delete_columns("account", 42) + + async def test_empty_column_name_raises(self): + """ValueError is raised when the column name string is empty.""" + client = _make_client() + with pytest.raises(ValueError, match="non-empty"): + await client._delete_columns("account", "") + + async def test_table_not_found_raises(self): + """MetadataError is raised when the parent table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._delete_columns("nonexistent", "col") + + async def test_column_not_found_raises(self): + """MetadataError is raised when the column is absent from attribute metadata.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": []}, status=200) + client._request.side_effect = [entity_resp, attr_resp] + with pytest.raises(MetadataError, match="not found"): + await client._delete_columns("account", "nonexistent_col") + + async def test_missing_attr_metadata_id_raises(self): + """RuntimeError is raised when the attribute response lacks a MetadataId.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"LogicalName": "new_notes"}]}, status=200) + client._request.side_effect = [entity_resp, attr_resp] + with pytest.raises(RuntimeError, match="MetadataId"): + await client._delete_columns("account", "new_Notes") + + async def test_picklist_column_flushes_cache(self): + """Deleting a Picklist-type column invalidates the picklist label cache.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp( + json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_status", "AttributeType": "Picklist"}]}, + status=200, + ) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + result = await client._delete_columns("account", "new_Status") + assert result == ["new_Status"] + assert key not in client._picklist_label_cache + + +# --------------------------------------------------------------------------- +# _list_columns() +# --------------------------------------------------------------------------- + + +class TestListColumns: + """Tests for _list_columns() attribute metadata listing.""" + + async def test_returns_attribute_list(self): + """The full attribute list from the API response is returned.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + cols_resp = _resp(json_data={"value": [{"LogicalName": "name"}, {"LogicalName": "accountid"}]}, status=200) + client._request.side_effect = [entity_resp, cols_resp] + result = await client._list_columns("account") + assert len(result) == 2 + + async def test_table_not_found_raises(self): + """MetadataError is raised when the table is absent from metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._list_columns("nonexistent") + + async def test_with_select_and_filter(self): + """Optional select and filter parameters are forwarded to the Attributes API call.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + cols_resp = _resp(json_data={"value": []}, status=200) + client._request.side_effect = [entity_resp, cols_resp] + result = await client._list_columns("account", select=["LogicalName"], filter="AttributeType eq 'String'") + assert result == [] + + +# --------------------------------------------------------------------------- +# Alternate key operations +# --------------------------------------------------------------------------- + + +class TestAlternateKeys: + """Tests for _create_alternate_key(), _get_alternate_keys(), and _delete_alternate_key().""" + + async def test_create_alternate_key_success(self): + """The key UUID is extracted from the OData-EntityId header and returned in metadata_id. + + The URL format is EntityDefinitions(LogicalName='...')/Keys(uuid), so the regex + skips the LogicalName= form and matches only the key UUID in parentheses. + """ + client = _make_client() + key_uuid = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + entity_resp = _resp(json_data=_entity_def(), status=200) + create_resp = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/EntityDefinitions(LogicalName='account')/Keys({key_uuid})" + }, + ) + client._request.side_effect = [entity_resp, create_resp] + result = await client._create_alternate_key("account", "new_prod_key", ["new_productcode"]) + assert result["schema_name"] == "new_prod_key" + assert result["key_attributes"] == ["new_productcode"] + assert result["metadata_id"] == key_uuid + + async def test_create_alternate_key_table_not_found_raises(self): + """MetadataError is raised when the target table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._create_alternate_key("nonexistent", "key", ["col"]) + + async def test_get_alternate_keys_returns_list(self): + """All alternate keys on the table are returned as a list of dicts.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + keys_resp = _resp(json_data={"value": [{"SchemaName": "key1"}, {"SchemaName": "key2"}]}, status=200) + client._request.side_effect = [entity_resp, keys_resp] + result = await client._get_alternate_keys("account") + assert len(result) == 2 + + async def test_get_alternate_keys_table_not_found_raises(self): + """MetadataError is raised when the table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._get_alternate_keys("nonexistent") + + async def test_delete_alternate_key_success(self): + """Two requests are issued: one entity lookup then one DELETE for the key.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, delete_resp] + await client._delete_alternate_key("account", "key-guid") + assert client._request.call_count == 2 + + async def test_delete_alternate_key_table_not_found_raises(self): + """MetadataError is raised when the table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._delete_alternate_key("nonexistent", "key-guid") + + +# --------------------------------------------------------------------------- +# _upsert() / _upsert_multiple() +# --------------------------------------------------------------------------- + + +class TestUpsert: + """Tests for _upsert() and _upsert_multiple() alternate-key upsert operations.""" + + async def test_upsert_issues_patch(self): + """_upsert() issues a PATCH request (create-or-replace semantics).""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._upsert("accounts", "account", {"accountnumber": "A"}, {"name": "X"}) + call = client._request.call_args + assert call.args[0] == "patch" + + async def test_upsert_multiple_issues_post(self): + """_upsert_multiple() sends a POST to the UpsertMultiple action endpoint.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}, {"accountnumber": "B"}], + [{"name": "X"}, {"name": "Y"}], + ) + call = client._request.call_args + assert call.args[0] == "post" + assert "UpsertMultiple" in call.args[1] + + async def test_upsert_multiple_mismatched_length_raises(self): + """ValueError is raised when the alternate-key list and record list differ in length.""" + client = _make_client() + with pytest.raises(ValueError, match="same length"): + await client._upsert_multiple("accounts", "account", [{"k": "1"}], [{"n": "A"}, {"n": "B"}]) + + async def test_upsert_multiple_key_conflict_raises(self): + """ValueError is raised when a record field conflicts with its alternate-key field.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValueError, match="conflicts"): + await client._upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"accountnumber": "B"}], + ) + + +# --------------------------------------------------------------------------- +# _bulk_fetch_picklists() / _convert_labels_to_ints() +# --------------------------------------------------------------------------- + + +class TestPicklists: + """Tests for _bulk_fetch_picklists() cache population and _convert_labels_to_ints() resolution.""" + + async def test_bulk_fetch_populates_cache(self): + """Picklist options are fetched from the API and stored with lowercased label keys.""" + client = _make_client() + body = { + "value": [ + { + "LogicalName": "statecode", + "OptionSet": { + "Options": [ + {"Value": 0, "Label": {"LocalizedLabels": [{"Label": "Active", "LanguageCode": 1033}]}}, + {"Value": 1, "Label": {"LocalizedLabels": [{"Label": "Inactive", "LanguageCode": 1033}]}}, + ] + }, + } + ] + } + client._request.return_value = _resp(json_data=body, status=200) + await client._bulk_fetch_picklists("account") + key = client._normalize_cache_key("account") + assert key in client._picklist_label_cache + picklists = client._picklist_label_cache[key]["picklists"] + assert "statecode" in picklists + assert picklists["statecode"]["active"] == 0 + + async def test_bulk_fetch_skips_on_cache_hit(self): + """A valid cache entry prevents an API call when the TTL has not expired.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + await client._bulk_fetch_picklists("account") + client._request.assert_not_called() + + async def test_bulk_fetch_empty_option_set(self): + """An attribute with an empty OptionSet is stored as an empty mapping.""" + client = _make_client() + body = {"value": [{"LogicalName": "field", "OptionSet": {}}]} + client._request.return_value = _resp(json_data=body, status=200) + await client._bulk_fetch_picklists("account") + key = client._normalize_cache_key("account") + assert client._picklist_label_cache[key]["picklists"]["field"] == {} + + async def test_convert_no_string_values_returns_unchanged(self): + """A record with no string values is returned as-is without any API lookup.""" + client = _make_client() + record = {"statecode": 0, "count": 5} + result = await client._convert_labels_to_ints("account", record) + assert result == record + client._request.assert_not_called() + + async def test_convert_string_resolved_to_int(self): + """A known label string is resolved to its integer option value from the cache.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"statecode": {"active": 0, "inactive": 1}}, + } + result = await client._convert_labels_to_ints("account", {"statecode": "Active"}) + assert result["statecode"] == 0 + + async def test_convert_odata_key_skipped(self): + """OData annotation fields with labels that don't match any option are left unchanged.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"@odata.type": {"val": 1}}, + } + record = {"@odata.type": "Microsoft.Dynamics.CRM.account"} + result = await client._convert_labels_to_ints("account", record) + assert result["@odata.type"] == "Microsoft.Dynamics.CRM.account" + + async def test_convert_unresolved_string_left_unchanged(self): + """A string value with no matching picklist entry is left as-is in the output.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + result = await client._convert_labels_to_ints("account", {"name": "Contoso"}) + assert result["name"] == "Contoso" + + +# --------------------------------------------------------------------------- +# _build_* async methods +# --------------------------------------------------------------------------- + + +class TestBuildMethods: + """Tests for _build_* async methods that produce _RawRequest objects without I/O.""" + + async def test_build_create_post_request(self): + """_build_create() produces a POST request targeting the entity set URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create("accounts", "account", {"amount": 100}) + assert req.method == "POST" + assert "accounts" in req.url + + async def test_build_create_multiple_post_request(self): + """_build_create_multiple() produces a POST targeting the CreateMultiple action.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create_multiple("accounts", "account", [{"amount": 100}]) + assert req.method == "POST" + assert "CreateMultiple" in req.url + + async def test_build_create_multiple_injects_odata_type(self): + """Each entry in the Targets list receives an @odata.type annotation.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create_multiple("accounts", "account", [{"amount": 100}]) + body = json.loads(req.body) + assert "@odata.type" in body["Targets"][0] + + async def test_build_update_patch_request(self): + """_build_update() produces a PATCH request with an If-Match: * concurrency guard.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update("account", "guid-1", {"name": "X"}) + assert req.method == "PATCH" + assert "accounts" in req.url + assert req.headers.get("If-Match") == "*" + + async def test_build_update_with_content_id_reference(self): + """A $-prefixed record_id is used as a raw changeset content-ID reference URL.""" + client = _make_client() + req = await client._build_update("account", "$1", {"name": "X"}) + assert req.url == "$1" + + async def test_build_delete_delete_request(self): + """_build_delete() produces a DELETE request with an If-Match: * concurrency guard.""" + client = _make_client() + _seed_cache(client) + req = await client._build_delete("account", "guid-1") + assert req.method == "DELETE" + assert "accounts" in req.url + assert req.headers.get("If-Match") == "*" + + async def test_build_delete_with_content_id_reference(self): + """A $-prefixed record_id is used as a raw changeset content-ID reference URL.""" + client = _make_client() + req = await client._build_delete("account", "$2") + assert req.url == "$2" + + async def test_build_get_get_request_with_select(self): + """_build_get() encodes the select list as a $select query string parameter.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1", select=["name", "telephone1"]) + assert req.method == "GET" + assert "accounts" in req.url + assert "$select=name,telephone1" in req.url + + async def test_build_get_no_select(self): + """_build_get() omits $select from the URL when no columns are specified.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1") + assert "$select" not in req.url + + async def test_build_sql_encodes_query(self): + """_build_sql() produces a GET request with the SQL statement in a sql= parameter.""" + client = _make_client() + _seed_cache(client) + req = await client._build_sql("SELECT name FROM account") + assert req.method == "GET" + assert "sql=" in req.url + assert "SELECT" in req.url or "SELECT" in req.url.replace("%20", " ") + + async def test_build_upsert_patch_request(self): + """_build_upsert() produces a PATCH without If-Match, allowing create-or-replace semantics.""" + client = _make_client() + req = await client._build_upsert("accounts", "account", {"accountnumber": "A"}, {"name": "X"}) + assert req.method == "PATCH" + assert "accounts" in req.url + assert req.headers is None or "If-Match" not in req.headers + + async def test_build_upsert_multiple_post_request(self): + """_build_upsert_multiple() produces a POST targeting the UpsertMultiple action.""" + client = _make_client() + req = await client._build_upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"name": "X"}], + ) + assert req.method == "POST" + assert "UpsertMultiple" in req.url + + async def test_build_upsert_multiple_mismatched_raises(self): + """ValidationError is raised when the alternate-key and record lists differ in length.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._build_upsert_multiple("accounts", "account", [{"k": "1"}], [{"n": "A"}, {"n": "B"}]) + + async def test_build_upsert_multiple_key_conflict_raises(self): + """ValidationError is raised when a record field overwrites an alternate-key field.""" + client = _make_client() + with pytest.raises(ValidationError, match="conflicts"): + await client._build_upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"accountnumber": "B"}], + ) + + async def test_build_delete_multiple_bulk_delete(self): + """_build_delete_multiple() produces a POST BulkDelete request with a QuerySet body.""" + client = _make_client() + _seed_cache(client) + req = await client._build_delete_multiple("account", ["id-1", "id-2"]) + assert req.method == "POST" + assert "BulkDelete" in req.url + body = json.loads(req.body) + assert "QuerySet" in body + + async def test_build_update_multiple_broadcast(self): + """A dict for changes is broadcast to all IDs; Targets list length matches ID count.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update_multiple("accounts", "account", ["id-1", "id-2"], {"name": "X"}) + assert req.method == "POST" + assert "UpdateMultiple" in req.url + body = json.loads(req.body) + assert len(body["Targets"]) == 2 + + async def test_build_update_multiple_paired(self): + """A list for changes is applied pairwise; Targets list matches the paired length.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update_multiple("accounts", "account", ["id-1"], [{"name": "X"}]) + assert req.method == "POST" + body = json.loads(req.body) + assert len(body["Targets"]) == 1 + + async def test_build_update_multiple_invalid_changes_type_raises(self): + """ValidationError is raised when changes is neither a dict nor a list.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValidationError): + await client._build_update_multiple("accounts", "account", ["id-1"], "invalid") + + async def test_build_update_multiple_mismatched_length_raises(self): + """ValidationError is raised when the ID list and changes list differ in length.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValidationError): + await client._build_update_multiple("accounts", "account", ["id-1", "id-2"], [{"name": "X"}]) + + +# --------------------------------------------------------------------------- +# _wait_for_attribute_visibility() +# --------------------------------------------------------------------------- + + +class TestWaitForAttributeVisibility: + """Tests for _wait_for_attribute_visibility() polling loop.""" + + async def test_succeeds_on_first_attempt(self): + """Returns immediately when the first probe request succeeds.""" + client = _make_client() + client._request.return_value = _resp(status=200) + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0,)) + client._request.assert_called_once() + + async def test_raises_after_all_delays_exhausted(self): + """RuntimeError is raised when every probe attempt fails and delays are exhausted.""" + client = _make_client() + client._request.side_effect = Exception("not visible") + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(RuntimeError, match="did not become visible"): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0, 0)) + + async def test_succeeds_after_retry(self): + """A transient failure on the first probe does not prevent success on the second.""" + client = _make_client() + client._request.side_effect = [Exception("not ready"), _resp(status=200)] + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0, 0)) + assert client._request.call_count == 2 + + +# --------------------------------------------------------------------------- +# _request_metadata_with_retry() +# --------------------------------------------------------------------------- + + +class TestRequestMetadataWithRetry: + """Tests for _request_metadata_with_retry() which retries on transient 404 responses.""" + + async def test_success_on_first_attempt(self): + """A successful response is returned immediately without any retry.""" + client = _make_client() + client._request.return_value = _resp(status=200, json_data={"value": []}) + result = await client._request_metadata_with_retry("get", "https://example/url") + assert result.status == 200 + + async def test_non_404_raises_immediately(self): + """A non-404 HttpError is re-raised without retrying.""" + client = _make_client() + err = HttpError("Server error", status_code=500) + client._request.side_effect = err + with pytest.raises(HttpError): + await client._request_metadata_with_retry("get", "https://example/url") + assert client._request.call_count == 1 + + async def test_404_retries_and_raises_runtime_error(self): + """A 404 is retried max_attempts=5 times before RuntimeError is raised.""" + client = _make_client() + err = HttpError("Not found", status_code=404) + client._request.side_effect = err + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(RuntimeError, match="Metadata request failed"): + await client._request_metadata_with_retry("get", "https://example/url") + assert client._request.call_count == 5 # max_attempts defined in implementation + + +# --------------------------------------------------------------------------- +# Additional coverage tests +# --------------------------------------------------------------------------- + + +class TestRequestMergeAndEdgeCases: + """Coverage for _request() header-merge, text-decode failure, and Retry-After edge cases.""" + + def _auth_client(self): + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + return _AsyncODataClient(auth, "https://example.crm.dynamics.com") + + async def test_caller_headers_merged_with_base_headers(self): + """Headers passed by the caller are merged on top of base headers.""" + client = self._auth_client() + client._raw_request = AsyncMock(return_value=_resp(status=200, json_data={})) + await client._request( + "get", "https://example.crm.dynamics.com/api/data/v9.2/accounts", headers={"X-Custom": "value"} + ) + _, kwargs = client._raw_request.call_args + assert kwargs.get("headers", {}).get("X-Custom") == "value" + assert "Authorization" in kwargs.get("headers", {}) + + async def test_non_json_body_still_raises_http_error(self): + """When r.text is non-JSON, _request still raises HttpError with the status code.""" + client = self._auth_client() + r = MagicMock() + r.status = 400 + r.headers = {} + r.text = "not valid json \xff" + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 400 + + async def test_retry_after_non_integer_handled(self): + """A non-integer Retry-After header (e.g. HTTP-date) does not crash _request.""" + client = self._auth_client() + body = {"error": {"code": "429", "message": "Too many requests"}} + r = _resp(status=429, json_data=body, headers={"Retry-After": "Wed, 21 Oct 2025 07:28:00 GMT"}) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("retry_after") is None + + +class TestCreateMultipleEdgeCases: + """Coverage for _create_multiple() JSON-parse and non-dict body paths.""" + + async def test_json_parse_failure_returns_empty_list(self): + """When response JSON cannot be parsed, returns empty list without raising.""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + async def test_non_dict_body_returns_empty_list(self): + """When response body is a list (not dict), returns empty list.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data=[1, 2, 3], status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + +class TestGetMultipleEdgeCases: + """Coverage for _get_multiple() JSON-parse failure path.""" + + async def test_json_parse_failure_returns_empty_page(self): + """When _do_request JSON parse fails, an empty dict is returned (no crash).""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + pages = [page async for page in client._get_multiple("account")] + assert pages == [] + + +class TestQuerySqlEdgeCases: + """Coverage for _query_sql() JSON-parse, non-dict body, and pagination error paths.""" + + async def test_json_parse_failure_returns_empty_list(self): + """When JSON parse fails on first response, returns empty list.""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._query_sql("SELECT name FROM account") + assert result == [] + + async def test_non_dict_body_returns_empty_list(self): + """When response body is not a dict (e.g. bare list of non-dicts), returns [].""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data="not-a-dict", status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == [] + + async def test_pagination_duplicate_cookie_warns_and_stops(self): + """Duplicate pagingcookie in $skiptoken triggers RuntimeWarning and stops pagination.""" + import warnings + from urllib.parse import quote + + client = _make_client() + _seed_cache(client) + # Build two skiptokens with the same pagingcookie value but different pagenumbers. + # _extract_pagingcookie extracts the pagingcookie= attribute value; if it's the + # same in both pages, the duplicate-cookie guard fires. + cookie_val = "%3Ccookie+guid%3D%22abc%22%3E" # same encoded cookie in both pages + outer1 = f'' + outer2 = f'' + next_link1 = f"https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken={quote(outer1)}" + next_link2 = f"https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken={quote(outer2)}" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link1, + }, + status=200, + ) + page2 = _resp( + json_data={ + "value": [{"name": "B", "accountid": "g2"}], + "@odata.nextLink": next_link2, + }, + status=200, + ) + client._request.side_effect = [page1, page2] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("pagingcookie" in str(warning.message) for warning in w) + assert len(result) >= 1 + + async def test_pagination_next_page_request_fails_warns_and_stops(self): + """When the next-page request raises, a RuntimeWarning is emitted and pagination stops.""" + import warnings + + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + client._request.side_effect = [page1, HttpError("server error", status_code=500)] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("next-page request failed" in str(warning.message) for warning in w) + assert result == [{"name": "A", "accountid": "g1"}] + + async def test_pagination_next_page_non_json_warns_and_stops(self): + """When the next-page response is not JSON, a RuntimeWarning is emitted.""" + import warnings + + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + bad_resp = MagicMock() + bad_resp.status = 200 + bad_resp.headers = {} + bad_resp.json = MagicMock(side_effect=ValueError("not json")) + client._request.side_effect = [page1, bad_resp] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("not valid JSON" in str(warning.message) for warning in w) + assert result == [{"name": "A", "accountid": "g1"}] + + async def test_pagination_non_dict_page_body_stops(self): + """When a paginated response body is not a dict, pagination stops cleanly.""" + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + page2 = _resp(json_data="not-a-dict", status=200) + client._request.side_effect = [page1, page2] + result = await client._query_sql("SELECT name FROM account") + assert result == [{"name": "A", "accountid": "g1"}] + + +class TestPrimaryIdAttrEdgeCases: + """Coverage for _primary_id_attr() RuntimeError when metadata lacks PrimaryIdAttribute.""" + + async def test_raises_when_pk_not_in_cache_after_metadata_fetch(self): + """RuntimeError raised when entity resolves but PrimaryIdAttribute is absent from cache.""" + client = _make_client() + # Populate entity set cache but NOT the primaryid cache + key = client._normalize_cache_key("account") + client._logical_to_entityset_cache[key] = "accounts" + # _entity_set_from_schema_name will hit the cache and return without populating primaryid + with pytest.raises(RuntimeError, match="PrimaryIdAttribute not resolved"): + await client._primary_id_attr("account") + + +class TestGetAttributeMetadataEdgeCases: + """Coverage for _get_attribute_metadata() skip and JSON-parse-failure paths.""" + + async def test_skips_at_sign_fields_in_extra_select(self): + """Fields starting with '@' in extra_select are silently ignored.""" + client = _make_client() + client._request.return_value = _resp( + json_data={"value": [{"MetadataId": "m1", "LogicalName": "name", "SchemaName": "Name"}]}, + status=200, + ) + result = await client._get_attribute_metadata("meta-1", "name", extra_select="@odata.type,AttributeType") + assert result is not None + assert result["LogicalName"] == "name" + + async def test_json_parse_failure_returns_none(self): + """When response JSON parse fails, None is returned without raising.""" + client = _make_client() + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._get_attribute_metadata("meta-1", "name") + assert result is None + + +class TestPicklistEdgeCases: + """Coverage for _bulk_fetch_picklists() guard clauses and _convert_labels_to_ints() paths.""" + + async def test_bulk_fetch_cache_hit_inside_lock_skips_fetch(self): + """Second TTL check inside the lock exits early when another coroutine populated cache.""" + import time + + client = _make_client() + key = client._normalize_cache_key("account") + # Pre-populate cache with a fresh entry (TTL not expired) + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + # Warm the outer cache check too + client._picklist_label_cache[key]["ts"] = time.time() + # Should return without calling _request + await client._bulk_fetch_picklists("account") + client._request_metadata_with_retry = AsyncMock() + client._request_metadata_with_retry.assert_not_called() + + async def test_bulk_fetch_skips_non_dict_items(self): + """Non-dict items in the picklist response value list are skipped.""" + client = _make_client() + r = _resp(json_data={"value": ["not-a-dict", {"LogicalName": "status", "OptionSet": {"Options": []}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_empty_logical_name(self): + """Items with empty LogicalName are skipped during picklist fetch.""" + client = _make_client() + r = _resp(json_data={"value": [{"LogicalName": "", "OptionSet": {"Options": []}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_non_dict_options(self): + """Non-dict entries in OptionSet.Options are skipped.""" + client = _make_client() + r = _resp(json_data={"value": [{"LogicalName": "status", "OptionSet": {"Options": ["bad"]}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_non_int_value(self): + """Options whose Value is not an int are skipped.""" + client = _make_client() + r = _resp( + json_data={ + "value": [ + { + "LogicalName": "status", + "OptionSet": {"Options": [{"Value": "not-an-int", "Label": {}}]}, + } + ] + } + ) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_convert_labels_non_dict_cache_entry_returns_record(self): + """When picklist cache entry is not a dict, record is returned unchanged.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = "not-a-dict" + result = await client._convert_labels_to_ints("account", {"status": "Active"}) + assert result == {"status": "Active"} + + async def test_convert_labels_skips_odata_annotation_keys(self): + """Keys containing '@odata.' are not looked up in the picklist cache.""" + import time + + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"status": {"active": 1}}, + } + record = {"status": "active", "status@odata.type": "#Microsoft.Dynamics.CRM.StatusType"} + result = await client._convert_labels_to_ints("account", record) + # status resolved; odata annotation key left untouched + assert result["status"] == 1 + assert "status@odata.type" in result + + +class TestCreateEntityEdgeCases: + """Coverage for _create_entity() solution_name, missing EntitySetName, missing MetadataId.""" + + async def test_create_entity_with_solution_unique_name(self): + """solution_unique_name is passed as a query parameter to the POST request.""" + client = _make_client() + client._request.return_value = _resp(status=204) + entity_resp = { + "LogicalName": "new_table", + "EntitySetName": "new_tables", + "MetadataId": "meta-999", + "SchemaName": "new_table", + "PrimaryIdAttribute": "new_tableid", + } + client._get_entity_by_table_schema_name = AsyncMock(return_value=entity_resp) + result = await client._create_entity( + "new_table", + "New Table", + [], + solution_unique_name="MySolution", + ) + _, kwargs = client._request.call_args + assert kwargs.get("params", {}).get("SolutionUniqueName") == "MySolution" + assert result["EntitySetName"] == "new_tables" + + async def test_create_entity_missing_entity_set_name_raises(self): + """RuntimeError raised when EntitySetName is absent after create.""" + client = _make_client() + client._request.return_value = _resp(status=204) + client._get_entity_by_table_schema_name = AsyncMock(return_value={"MetadataId": "m1"}) + with pytest.raises(RuntimeError, match="EntitySetName not available"): + await client._create_entity("t", "t", "T", []) + + async def test_create_entity_missing_metadata_id_raises(self): + """RuntimeError raised when MetadataId is absent after create.""" + client = _make_client() + client._request.return_value = _resp(status=204) + client._get_entity_by_table_schema_name = AsyncMock(return_value={"EntitySetName": "ts", "LogicalName": "t"}) + with pytest.raises(RuntimeError, match="MetadataId missing"): + await client._create_entity("t", "t", "T", []) + + +class TestWaitForAttributeVisibilityWithDelay: + """Coverage for _wait_for_attribute_visibility() sleep branch.""" + + async def test_waits_when_delay_is_nonzero(self): + """asyncio.sleep is called when the computed delay is positive.""" + client = _make_client() + # First call (delay=0) fails so the loop continues to delay=1 where sleep fires. + ok = _resp(status=200) + client._request.side_effect = [HttpError("not yet", status_code=404), ok] + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await client._wait_for_attribute_visibility("accounts", "new_col", delays=(0, 1)) + mock_sleep.assert_called_once_with(1) + + +class TestAlternateKeyWithDisplayName: + """Coverage for _create_alternate_key() display_name_label path.""" + + async def test_create_alternate_key_with_display_name(self): + """DisplayName payload key is set when display_name_label is provided.""" + client = _make_client() + ent = {"LogicalName": "account", "EntitySetName": "accounts", "MetadataId": "m1", "SchemaName": "Account"} + client._get_entity_by_table_schema_name = AsyncMock(return_value=ent) + r = _resp(status=204, headers={"OData-EntityId": "https://example.com/(key123)"}) + r.headers = {"OData-EntityId": "https://example.com/(key123)"} + client._request.return_value = r + + label = MagicMock() + label.to_dict.return_value = {"UserLocalizedLabel": {"Label": "Account Number", "LanguageCode": 1033}} + + result = await client._create_alternate_key("account", "AccountNumber_AK", ["accountnumber"], label) + assert result["schema_name"] == "AccountNumber_AK" + _, kwargs = client._request.call_args + assert "DisplayName" in kwargs.get("json", {}) + + +class TestBuildMethodsAdditional: + """Coverage for _build_create_multiple TypeError, _build_get annotations, and _build_list.""" + + async def test_build_create_multiple_non_dict_raises_type_error(self): + """_build_create_multiple() raises TypeError when records contain non-dicts.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError, match="dicts"): + await client._build_create_multiple("accounts", "account", ["not-a-dict"]) + + async def test_build_get_with_include_annotations(self): + """_build_get() sets Prefer header when include_annotations is specified.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1", include_annotations="*") + assert req.headers is not None + assert "odata.include-annotations" in req.headers.get("Prefer", "") + + async def test_build_list_basic(self): + """_build_list() produces a GET request targeting the entity-set URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account") + assert req.method == "GET" + assert "accounts" in req.url + assert req.headers is None + + async def test_build_list_with_select_filter_orderby_top(self): + """_build_list() encodes all OData query parameters into the URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list( + "account", + select=["name", "telephone1"], + filter="statecode eq 0", + orderby=["name asc"], + top=10, + ) + assert "$select=name,telephone1" in req.url + assert "$filter=statecode+eq+0" in req.url or "$filter=statecode%20eq%200" in req.url or "statecode" in req.url + assert "$top=10" in req.url + + async def test_build_list_with_page_size_and_annotations(self): + """_build_list() sets Prefer header for page_size and include_annotations.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account", page_size=50, include_annotations="*") + assert req.headers is not None + prefer = req.headers.get("Prefer", "") + assert "odata.maxpagesize=50" in prefer + assert "odata.include-annotations" in prefer + + async def test_build_list_with_count(self): + """_build_list() appends $count=true when count=True.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account", count=True) + assert "$count=true" in req.url + + +class TestAsyncOperationContextUserAgent: + """User-Agent header reflects operation_context on the async client.""" + + async def test_default_user_agent_unchanged(self): + from PowerPlatform.Dataverse.aio.data._async_odata import _USER_AGENT + + client = _make_client() + headers = await client._headers() + assert headers["User-Agent"] == _USER_AGENT + + async def test_operation_context_appended(self): + from PowerPlatform.Dataverse.aio.data._async_odata import _USER_AGENT + from PowerPlatform.Dataverse.core.config import DataverseConfig, OperationContext + + ctx_str = "app=dataverse-skills/1.2.1;agent=claude-code" + config = DataverseConfig(operation_context=OperationContext(user_agent_context=ctx_str)) + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com", config=config) + headers = await client._headers() + assert headers["User-Agent"] == f"{_USER_AGENT} ({ctx_str})" + + async def test_none_context_no_parentheses(self): + from PowerPlatform.Dataverse.core.config import DataverseConfig + + config = DataverseConfig(operation_context=None) + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com", config=config) + headers = await client._headers() + assert "(" not in headers["User-Agent"] diff --git a/tests/unit/aio/data/test_async_relationships.py b/tests/unit/aio/data/test_async_relationships.py new file mode 100644 index 00000000..cf0ed1bd --- /dev/null +++ b/tests/unit/aio/data/test_async_relationships.py @@ -0,0 +1,314 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncRelationshipOperationsMixin.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient +from PowerPlatform.Dataverse.core.errors import MetadataError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + ManyToManyRelationshipMetadata, + OneToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(json_data=None, status=200, headers=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.json = MagicMock(return_value=json_data if json_data is not None else {}) + return r + + +def _entity_def(meta_id="meta-001", logical="account"): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": "accounts", + "PrimaryIdAttribute": "accountid", + "MetadataId": meta_id, + "SchemaName": "Account", + } + ] + } + + +def _label(text: str = "Test") -> Label: + """Return a Label with a single English localized label.""" + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +def _seed_cache(client, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +# --------------------------------------------------------------------------- +# _extract_id_from_header (sync) +# --------------------------------------------------------------------------- + + +class TestExtractIdFromHeader: + """Tests for _extract_id_from_header(), which parses GUIDs from OData-EntityId URLs. + + The regex matches only hex characters and dashes inside parentheses, so + only proper UUID-format strings are extracted. + """ + + def test_extracts_guid_from_url(self): + """A UUID enclosed in parentheses at the end of a URL is returned.""" + client = _make_client() + guid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + header = f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + result = client._extract_id_from_header(header) + assert result == guid + + def test_returns_none_for_empty_header(self): + """None is returned for both None and empty-string inputs.""" + client = _make_client() + assert client._extract_id_from_header(None) is None + assert client._extract_id_from_header("") is None + + def test_returns_none_when_no_guid(self): + """None is returned when the header contains no hex UUID in parentheses.""" + client = _make_client() + assert client._extract_id_from_header("no-guid-here") is None + + +# --------------------------------------------------------------------------- +# _create_one_to_many_relationship() +# --------------------------------------------------------------------------- + + +class TestCreateOneToManyRelationship: + """Tests for _create_one_to_many_relationship() one-to-many relationship creation.""" + + async def test_success(self): + """The relationship ID, schema name, and lookup schema name are returned on success.""" + client = _make_client() + guid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + client._request.return_value = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + }, + ) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Dept")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + result = await client._create_one_to_many_relationship(lookup, relationship) + assert result["relationship_id"] == guid + assert result["relationship_schema_name"] == "new_Dept_Emp" + assert result["lookup_schema_name"] == "new_DeptId" + + async def test_with_solution(self): + """The MSCRM.SolutionUniqueName header is injected when a solution name is supplied.""" + client = _make_client() + client._request.return_value = _resp(status=204, headers={}) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Dept")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + await client._create_one_to_many_relationship(lookup, relationship, solution="MySolution") + call_kwargs = client._request.call_args.kwargs + headers = call_kwargs.get("headers", {}) + assert "MSCRM.SolutionUniqueName" in headers + assert headers["MSCRM.SolutionUniqueName"] == "MySolution" + + +# --------------------------------------------------------------------------- +# _create_many_to_many_relationship() +# --------------------------------------------------------------------------- + + +class TestCreateManyToManyRelationship: + """Tests for _create_many_to_many_relationship() many-to-many relationship creation.""" + + async def test_success(self): + """The relationship ID and entity names are returned on success.""" + client = _make_client() + guid = "b2c3d4e5-f6a7-8901-bcde-f12345678901" + client._request.return_value = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + }, + ) + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + result = await client._create_many_to_many_relationship(relationship) + assert result["relationship_id"] == guid + assert result["relationship_schema_name"] == "new_emp_proj" + assert result["entity1_logical_name"] == "new_employee" + assert result["entity2_logical_name"] == "new_project" + + async def test_with_solution(self): + """The MSCRM.SolutionUniqueName header is injected when a solution name is supplied.""" + client = _make_client() + client._request.return_value = _resp(status=204, headers={}) + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + await client._create_many_to_many_relationship(relationship, solution="MySol") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("MSCRM.SolutionUniqueName") == "MySol" + + +# --------------------------------------------------------------------------- +# _delete_relationship() +# --------------------------------------------------------------------------- + + +class TestDeleteRelationship: + """Tests for _delete_relationship() relationship removal by GUID.""" + + async def test_issues_delete(self): + """A DELETE request is issued containing the relationship GUID in the URL.""" + client = _make_client() + client._request.return_value = _resp(status=204) + await client._delete_relationship("rel-guid-1") + call_args = client._request.call_args + assert call_args.args[0] == "delete" + assert "rel-guid-1" in call_args.args[1] + + async def test_sets_if_match_header(self): + """An If-Match: * header is sent to prevent accidental deletion of a stale version.""" + client = _make_client() + client._request.return_value = _resp(status=204) + await client._delete_relationship("rel-guid-1") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("If-Match") == "*" + + +# --------------------------------------------------------------------------- +# _get_relationship() +# --------------------------------------------------------------------------- + + +class TestGetRelationship: + """Tests for _get_relationship() single-relationship lookup by schema name.""" + + async def test_returns_relationship_dict(self): + """The first matching relationship dict from the value list is returned.""" + client = _make_client() + rel = {"SchemaName": "new_Dept_Emp", "RelationshipId": "rel-1"} + client._request.return_value = _resp(json_data={"value": [rel]}) + result = await client._get_relationship("new_Dept_Emp") + assert result == rel + + async def test_returns_none_when_not_found(self): + """None is returned when the API returns an empty value list.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + result = await client._get_relationship("nonexistent") + assert result is None + + +# --------------------------------------------------------------------------- +# _list_relationships() +# --------------------------------------------------------------------------- + + +class TestListRelationships: + """Tests for _list_relationships() global relationship listing.""" + + async def test_returns_all_relationships(self): + """The full value list is returned when no filter is applied.""" + client = _make_client() + rels = [{"SchemaName": "rel1"}, {"SchemaName": "rel2"}] + client._request.return_value = _resp(json_data={"value": rels}) + result = await client._list_relationships() + assert result == rels + + async def test_with_filter_and_select(self): + """Optional filter and select parameters are forwarded as OData query params.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + result = await client._list_relationships( + filter="RelationshipType eq 'OneToMany'", + select=["SchemaName"], + ) + assert result == [] + call_kwargs = client._request.call_args.kwargs + params = call_kwargs.get("params", {}) + assert "$filter" in params + assert "$select" in params + + +# --------------------------------------------------------------------------- +# _list_table_relationships() +# --------------------------------------------------------------------------- + + +class TestListTableRelationships: + """Tests for _list_table_relationships() which aggregates all three relationship types.""" + + async def test_combines_three_relationship_types(self): + """One-to-many, many-to-one, and many-to-many relationships are combined into one list.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def()) + otm_resp = _resp(json_data={"value": [{"SchemaName": "rel_otm"}]}) + mto_resp = _resp(json_data={"value": [{"SchemaName": "rel_mto"}]}) + mtm_resp = _resp(json_data={"value": [{"SchemaName": "rel_mtm"}]}) + client._request.side_effect = [entity_resp, otm_resp, mto_resp, mtm_resp] + result = await client._list_table_relationships("account") + assert len(result) == 3 + schema_names = [r["SchemaName"] for r in result] + assert "rel_otm" in schema_names + assert "rel_mto" in schema_names + assert "rel_mtm" in schema_names + + async def test_table_not_found_raises(self): + """MetadataError is raised when the table does not exist in entity metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + with pytest.raises(MetadataError, match="not found"): + await client._list_table_relationships("nonexistent") + + async def test_with_filter_and_select(self): + """Optional filter and select parameters are forwarded to all three relationship requests.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def()) + empty_resp = _resp(json_data={"value": []}) + client._request.side_effect = [entity_resp, empty_resp, empty_resp, empty_resp] + result = await client._list_table_relationships( + "account", + filter="IsCustomRelationship eq true", + select=["SchemaName"], + ) + assert result == [] diff --git a/tests/unit/aio/data/test_async_upload.py b/tests/unit/aio/data/test_async_upload.py new file mode 100644 index 00000000..d5268913 --- /dev/null +++ b/tests/unit/aio/data/test_async_upload.py @@ -0,0 +1,318 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncFileUploadMixin.""" + +import os +import tempfile +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(status=200, headers=None, json_data=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.json = MagicMock(return_value=json_data or {}) + return r + + +def _seed_cache(client, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +def _entity_def(meta_id="meta-001", entity_set="accounts", logical="account"): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": entity_set, + "PrimaryIdAttribute": "accountid", + "MetadataId": meta_id, + "SchemaName": "Account", + } + ] + } + + +# --------------------------------------------------------------------------- +# _upload_file_small() +# --------------------------------------------------------------------------- + + +class TestUploadFileSmall: + """Tests for _upload_file_small(), the single-request upload path for small files.""" + + async def test_success_uploads_with_patch(self): + """A successful upload issues a PATCH with x-ms-file-name and If-None-Match headers.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path) + call_args = client._request.call_args + assert call_args.args[0] == "patch" + headers = call_args.kwargs.get("headers", {}) + assert "x-ms-file-name" in headers + assert headers.get("If-None-Match") == "null" + finally: + os.unlink(path) + + async def test_success_with_overwrite(self): + """When if_none_match=False, an If-Match: * header is sent instead of If-None-Match.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path, if_none_match=False) + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("If-Match") == "*" + assert "If-None-Match" not in headers + finally: + os.unlink(path) + + async def test_explicit_mime_type(self): + """An explicit content_type is forwarded as the Content-Type header.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as f: + f.write(b"%PDF") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path, content_type="application/pdf") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("Content-Type") == "application/pdf" + finally: + os.unlink(path) + + async def test_empty_record_id_raises(self): + """ValueError is raised immediately when record_id is an empty string.""" + client = _make_client() + with pytest.raises(ValueError, match="record_id required"): + await client._upload_file_small("accounts", "", "new_doc", "/any/path") + + async def test_file_not_found_raises(self): + """FileNotFoundError is raised when the specified file path does not exist.""" + client = _make_client() + with pytest.raises(FileNotFoundError): + await client._upload_file_small("accounts", "guid-1", "new_doc", "/nonexistent/path.txt") + + async def test_file_too_large_raises(self): + """ValueError is raised when the file size exceeds the single-upload size limit.""" + client = _make_client() + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"x") + path = f.name + try: + stat_result = MagicMock() + stat_result.st_size = 200 * 1024 * 1024 + stat_result.st_mode = 0o100644 # regular file + with patch("pathlib.Path.stat", return_value=stat_result): + with pytest.raises(ValueError, match="exceeds single-upload limit"): + await client._upload_file_small("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# _upload_file_chunk() +# --------------------------------------------------------------------------- + + +class TestUploadFileChunk: + """Tests for _upload_file_chunk(), the chunked upload path for large files.""" + + async def test_success_single_chunk(self): + """A small file completes in two requests: session init and one chunk PUT.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_document?sessiontoken=xyz" + init_resp = _resp(status=200, headers={"Location": location, "x-ms-chunk-size": "4194304"}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_document", path) + assert client._request.call_count == 2 + finally: + os.unlink(path) + + async def test_success_with_if_match(self): + """When if_none_match=False, an If-Match: * header is included in the session-init request.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_document?sessiontoken=abc" + init_resp = _resp(status=200, headers={"Location": location}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_document", path, if_none_match=False) + init_headers = client._request.call_args_list[0].kwargs.get("headers", {}) + assert init_headers.get("If-Match") == "*" + finally: + os.unlink(path) + + async def test_empty_record_id_raises(self): + """ValueError is raised immediately when record_id is an empty string.""" + client = _make_client() + with pytest.raises(ValueError, match="record_id required"): + await client._upload_file_chunk("accounts", "", "new_doc", "/any/path") + + async def test_file_not_found_raises(self): + """FileNotFoundError is raised when the specified file path does not exist.""" + client = _make_client() + with pytest.raises(FileNotFoundError): + await client._upload_file_chunk("accounts", "guid-1", "new_doc", "/nonexistent/path.txt") + + async def test_missing_location_header_raises(self): + """RuntimeError is raised when the session-init response lacks a Location header.""" + client = _make_client() + client._request.return_value = _resp(status=200, headers={}) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + with pytest.raises(RuntimeError, match="Missing Location header"): + await client._upload_file_chunk("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + async def test_invalid_chunk_size_falls_back_to_default(self): + """A non-integer x-ms-chunk-size header is ignored and the 4MB default is used.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_doc?tok=x" + init_resp = _resp(status=200, headers={"Location": location, "x-ms-chunk-size": "invalid"}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# _upload_file() — auto mode dispatch +# --------------------------------------------------------------------------- + + +class TestUploadFile: + """Tests for _upload_file(), the high-level dispatcher that selects the upload path.""" + + async def test_small_file_uses_small_mode(self): + """mode='small' routes to _upload_file_small without calling _upload_file_chunk.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + client._upload_file_small = AsyncMock(return_value=None) + client._upload_file_chunk = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"small content") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="small") + client._upload_file_small.assert_called_once() + client._upload_file_chunk.assert_not_called() + finally: + os.unlink(path) + + async def test_chunk_mode_uses_chunk_upload(self): + """mode='chunk' routes to _upload_file_chunk without calling _upload_file_small.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + client._upload_file_small = AsyncMock(return_value=None) + client._upload_file_chunk = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as f: + f.write(b"big content") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="chunk") + client._upload_file_chunk.assert_called_once() + finally: + os.unlink(path) + + async def test_invalid_mode_raises(self): + """ValueError is raised when an unrecognised mode string is supplied.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + with pytest.raises(ValueError, match="Invalid mode"): + await client._upload_file("account", "guid-1", "new_doc", path, mode="badmode") + finally: + os.unlink(path) + + async def test_auto_mode_file_not_found_raises(self): + """FileNotFoundError is raised in default auto mode when the file path does not exist.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + with pytest.raises(FileNotFoundError): + await client._upload_file("account", "guid-1", "new_doc", "/nonexistent/file.txt") + + async def test_attribute_not_found_creates_it(self): + """When attribute metadata is missing, _create_columns and _wait_for_attribute_visibility are called.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value=None) + client._create_columns = AsyncMock(return_value=["new_doc"]) + client._wait_for_attribute_visibility = AsyncMock(return_value=None) + client._upload_file_small = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="small") + client._create_columns.assert_called_once_with("account", {"new_doc": "file"}) + client._wait_for_attribute_visibility.assert_called_once() + finally: + os.unlink(path) diff --git a/tests/unit/aio/test_async_batch.py b/tests/unit/aio/test_async_batch.py new file mode 100644 index 00000000..baad3ee6 --- /dev/null +++ b/tests/unit/aio/test_async_batch.py @@ -0,0 +1,511 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +import pandas as pd +from unittest.mock import AsyncMock + + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_batch import ( + AsyncBatchOperations, + AsyncBatchRequest, + AsyncChangeSet, +) +from PowerPlatform.Dataverse.operations.batch import ( + BatchRecordOperations, + BatchTableOperations, + BatchQueryOperations, + BatchDataFrameOperations, + ChangeSetRecordOperations, +) +from PowerPlatform.Dataverse.data._batch_base import ( + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + + +def _label(text: str = "Test") -> Label: + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +from PowerPlatform.Dataverse.core.errors import ValidationError + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_batch(async_client: AsyncDataverseClient) -> AsyncBatchRequest: + return async_client.batch.new() + + +# --------------------------------------------------------------------------- +# Namespace tests +# --------------------------------------------------------------------------- + + +class TestAsyncBatchOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.batch, AsyncBatchOperations) + + def test_new_returns_batch_request(self, async_client): + batch = async_client.batch.new() + assert isinstance(batch, AsyncBatchRequest) + + def test_batch_request_namespaces(self, async_client): + batch = async_client.batch.new() + assert isinstance(batch.records, BatchRecordOperations) + assert isinstance(batch.tables, BatchTableOperations) + assert isinstance(batch.query, BatchQueryOperations) + assert isinstance(batch.dataframe, BatchDataFrameOperations) + + +# --------------------------------------------------------------------------- +# AsyncBatchRecordOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchRecordOperations: + def test_create_single_appends_record_create(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", {"name": "Contoso"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordCreate) + assert item.table == "account" + assert item.data == {"name": "Contoso"} + + def test_create_bulk_appends_record_create(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", [{"name": "A"}, {"name": "B"}]) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordCreate) + + def test_update_single_appends_record_update(self, async_client): + batch = _make_batch(async_client) + batch.records.update("account", "guid-1", {"name": "X"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordUpdate) + assert item.table == "account" + + def test_delete_single_appends_record_delete(self, async_client): + batch = _make_batch(async_client) + batch.records.delete("account", "guid-1") + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordDelete) + + def test_delete_bulk_appends_record_delete(self, async_client): + batch = _make_batch(async_client) + batch.records.delete("account", ["guid-1", "guid-2"]) + assert isinstance(batch._items[0], _RecordDelete) + + def test_get_appends_record_get(self, async_client): + batch = _make_batch(async_client) + with pytest.warns(DeprecationWarning): + batch.records.get("account", "guid-1", select=["name"]) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordGet) + assert item.table == "account" + assert item.record_id == "guid-1" + assert item.select == ["name"] + + def test_upsert_appends_record_upsert(self, async_client): + batch = _make_batch(async_client) + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "X"}) + batch.records.upsert("account", [item]) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordUpsert) + + def test_upsert_dict_item_normalized(self, async_client): + batch = _make_batch(async_client) + batch.records.upsert("account", [{"alternate_key": {"accountnumber": "ACC-001"}, "record": {"name": "X"}}]) + enqueued = batch._items[0] + assert isinstance(enqueued, _RecordUpsert) + assert isinstance(enqueued.items[0], UpsertItem) + + def test_upsert_empty_list_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.records.upsert("account", []) + + def test_upsert_invalid_item_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.records.upsert("account", [42]) + + +# --------------------------------------------------------------------------- +# AsyncBatchTableOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchTableOperations: + def test_create_appends_table_create(self, async_client): + batch = _make_batch(async_client) + batch.tables.create("new_Product", {"new_Price": "decimal"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _TableCreate) + assert item.table == "new_Product" + + def test_delete_appends_table_delete(self, async_client): + batch = _make_batch(async_client) + batch.tables.delete("new_Product") + assert isinstance(batch._items[0], _TableDelete) + + def test_get_appends_table_get(self, async_client): + batch = _make_batch(async_client) + batch.tables.get("new_Product") + assert isinstance(batch._items[0], _TableGet) + + def test_list_appends_table_list(self, async_client): + batch = _make_batch(async_client) + batch.tables.list() + assert isinstance(batch._items[0], _TableList) + + def test_add_columns_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.add_columns("new_Product", {"new_Notes": "string"}) + assert isinstance(batch._items[0], _TableAddColumns) + + def test_remove_columns_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.remove_columns("new_Product", "new_Notes") + assert isinstance(batch._items[0], _TableRemoveColumns) + + def test_create_one_to_many_appends(self, async_client): + batch = _make_batch(async_client) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Department")) + rel = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_emp", + referenced_attribute="new_deptid", + ) + batch.tables.create_one_to_many_relationship(lookup, rel) + assert isinstance(batch._items[0], _TableCreateOneToMany) + + def test_create_many_to_many_appends(self, async_client): + batch = _make_batch(async_client) + rel = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_emp", + entity2_logical_name="new_proj", + ) + batch.tables.create_many_to_many_relationship(rel) + assert isinstance(batch._items[0], _TableCreateManyToMany) + + def test_delete_relationship_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.delete_relationship("rel-guid") + assert isinstance(batch._items[0], _TableDeleteRelationship) + + def test_get_relationship_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.get_relationship("new_Dept_Emp") + assert isinstance(batch._items[0], _TableGetRelationship) + + def test_create_lookup_field_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + ) + assert isinstance(batch._items[0], _TableCreateLookupField) + + +# --------------------------------------------------------------------------- +# AsyncBatchQueryOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchQueryOperations: + def test_sql_appends_query_sql(self, async_client): + batch = _make_batch(async_client) + batch.query.sql("SELECT name FROM account") + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _QuerySql) + assert item.sql == "SELECT name FROM account" + + def test_sql_strips_whitespace(self, async_client): + batch = _make_batch(async_client) + batch.query.sql(" SELECT name FROM account ") + assert batch._items[0].sql == "SELECT name FROM account" + + def test_sql_empty_string_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql("") + + def test_sql_whitespace_only_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql(" ") + + def test_sql_non_string_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql(None) + + +# --------------------------------------------------------------------------- +# AsyncBatchDataFrameOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchDataFrameOperations: + def test_create_from_dataframe(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + batch.dataframe.create("account", df) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordCreate) + + def test_create_non_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.create("account", [{"name": "X"}]) + + def test_create_empty_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.create("account", pd.DataFrame()) + + def test_create_all_null_row_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.create("account", pd.DataFrame([{"name": None}])) + + def test_update_from_dataframe(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1", "name": "X"}]) + batch.dataframe.update("account", df, id_column="accountid") + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordUpdate) + + def test_update_non_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.update("account", [{}], id_column="id") + + def test_update_empty_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.update("account", pd.DataFrame(), id_column="id") + + def test_update_missing_id_column_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"name": "X"}]) + with pytest.raises(ValueError, match="id_column"): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_invalid_ids_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": None, "name": "X"}]) + with pytest.raises(ValueError): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_no_change_columns_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1"}]) + with pytest.raises(ValueError): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_all_null_rows_skipped(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": None}]) + batch.dataframe.update("account", df, id_column="accountid") + # All change values null -> nothing enqueued + assert len(batch._items) == 0 + + def test_delete_from_series(self, async_client): + batch = _make_batch(async_client) + ids = pd.Series(["guid-1", "guid-2"]) + batch.dataframe.delete("account", ids) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordDelete) + + def test_delete_non_series_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.delete("account", ["guid-1"]) + + def test_delete_empty_series_no_item(self, async_client): + batch = _make_batch(async_client) + batch.dataframe.delete("account", pd.Series([], dtype=str)) + assert len(batch._items) == 0 + + def test_delete_invalid_ids_raises(self, async_client): + batch = _make_batch(async_client) + ids = pd.Series(["guid-1", None]) + with pytest.raises(ValueError): + batch.dataframe.delete("account", ids) + + +# --------------------------------------------------------------------------- +# AsyncChangeSet +# --------------------------------------------------------------------------- + + +class TestAsyncChangeSet: + def test_changeset_returns_async_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + assert isinstance(cs, AsyncChangeSet) + + def test_changeset_records_namespace(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + assert isinstance(cs.records, ChangeSetRecordOperations) + + def test_changeset_appended_to_items(self, async_client): + batch = _make_batch(async_client) + batch.changeset() + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _ChangeSet) + + async def test_changeset_async_context_manager(self, async_client): + batch = _make_batch(async_client) + async with batch.changeset() as cs: + assert isinstance(cs, AsyncChangeSet) + + +class TestAsyncChangeSetRecordOperations: + def test_create_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + ref = cs.records.create("account", {"name": "Contoso"}) + # ref should be a content-ID string like "$1" + assert isinstance(ref, str) + assert ref.startswith("$") + + def test_update_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + cs.records.update("account", "guid-1", {"name": "X"}) + internal = batch._items[0] + assert len(internal.operations) == 1 + + def test_delete_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + cs.records.delete("account", "guid-1") + internal = batch._items[0] + assert len(internal.operations) == 1 + + def test_content_id_increments(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + ref1 = cs.records.create("account", {"name": "A"}) + ref2 = cs.records.create("contact", {"firstname": "B"}) + assert ref1 != ref2 + + +# --------------------------------------------------------------------------- +# AsyncBatchRequest.execute +# --------------------------------------------------------------------------- + + +class TestAsyncBatchRequestExecute: + async def test_execute_calls_batch_client(self, async_client, mock_od): + """execute() delegates to _AsyncBatchClient and returns BatchResult.""" + from PowerPlatform.Dataverse.models.batch import BatchResult, BatchItemResponse + + mock_result = BatchResult(responses=[BatchItemResponse(status_code=204)]) + + # Patch _AsyncBatchClient so we don't need a real HTTP client + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + batch.records.create("account", {"name": "X"}) + result = await batch.execute() + + mock_instance.execute.assert_called_once() + assert isinstance(result, BatchResult) + + async def test_execute_passes_continue_on_error(self, async_client, mock_od): + """execute() passes continue_on_error to _AsyncBatchClient.execute.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + mock_result = BatchResult() + + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + await batch.execute(continue_on_error=True) + + _, kwargs = mock_instance.execute.call_args + assert kwargs["continue_on_error"] is True + + async def test_execute_empty_batch_ok(self, async_client, mock_od): + """execute() with an empty batch does not raise.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + mock_result = BatchResult() + + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + result = await batch.execute() + + assert isinstance(result, BatchResult) + + +# --------------------------------------------------------------------------- +# Multiple operations in one batch +# --------------------------------------------------------------------------- + + +class TestAsyncBatchMultipleOperations: + def test_multiple_items_accumulated(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", {"name": "A"}) + with pytest.warns(DeprecationWarning): + batch.records.get("account", "guid-1") + batch.tables.get("account") + batch.query.sql("SELECT name FROM account") + assert len(batch._items) == 4 diff --git a/tests/unit/aio/test_async_client.py b/tests/unit/aio/test_async_client.py new file mode 100644 index 00000000..5997a30a --- /dev/null +++ b/tests/unit/aio/test_async_client.py @@ -0,0 +1,241 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations +from PowerPlatform.Dataverse.aio.operations.async_dataframe import AsyncDataFrameOperations +from PowerPlatform.Dataverse.aio.operations.async_batch import AsyncBatchOperations +from PowerPlatform.Dataverse.core.config import DataverseConfig, OperationContext + + +def _make_credential() -> MagicMock: + return MagicMock(spec=AsyncTokenCredential) + + +class TestAsyncDataverseClientInit: + """Tests for AsyncDataverseClient initialization and validation.""" + + def test_valid_init(self): + """AsyncDataverseClient initializes with valid url and credential.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._base_url == "https://org.crm.dynamics.com" + assert not client._closed + + def test_trailing_slash_stripped(self): + """Trailing slash is stripped from base_url.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com/", _make_credential()) + assert client._base_url == "https://org.crm.dynamics.com" + + def test_empty_base_url_raises(self): + """Empty base_url raises ValueError.""" + with pytest.raises(ValueError, match="base_url is required"): + AsyncDataverseClient("", _make_credential()) + + def test_namespace_attributes_created(self): + """All operation namespace attributes are created.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert isinstance(client.records, AsyncRecordOperations) + assert isinstance(client.tables, AsyncTableOperations) + assert isinstance(client.query, AsyncQueryOperations) + assert isinstance(client.files, AsyncFileOperations) + assert isinstance(client.dataframe, AsyncDataFrameOperations) + assert isinstance(client.batch, AsyncBatchOperations) + + def test_odata_and_session_initially_none(self): + """_odata and _session are None until first use.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._odata is None + assert client._session is None + + +class TestAsyncDataverseClientContextManager: + """Tests for async context manager protocol.""" + + async def test_aenter_returns_self(self): + """__aenter__ returns the client instance.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session_cls.return_value = MagicMock() + result = await client.__aenter__() + assert result is client + + async def test_aenter_creates_session(self): + """__aenter__ creates an aiohttp.ClientSession.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session_cls.return_value = MagicMock() + await client.__aenter__() + mock_session_cls.assert_called_once() + assert client._session is not None + + async def test_aenter_does_not_recreate_existing_session(self): + """__aenter__ does not replace an existing session.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + existing_session = MagicMock() + client._session = existing_session + with patch("aiohttp.ClientSession") as mock_session_cls: + await client.__aenter__() + mock_session_cls.assert_not_called() + assert client._session is existing_session + + async def test_aexit_calls_aclose(self): + """__aexit__ calls aclose() to release resources.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client.aclose = AsyncMock() + await client.__aexit__(None, None, None) + client.aclose.assert_called_once() + + async def test_aenter_raises_after_close(self): + """__aenter__ raises RuntimeError after the client has been closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + await client.__aenter__() + + +class TestAsyncDataverseClientAclose: + """Tests for aclose() lifecycle.""" + + async def test_aclose_sets_closed_flag(self): + """aclose() marks the client as closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + await client.aclose() + assert client._closed + + async def test_aclose_closes_session(self): + """aclose() closes the aiohttp.ClientSession.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + mock_session = MagicMock() + mock_session.close = AsyncMock() + client._session = mock_session + await client.aclose() + mock_session.close.assert_called_once() + assert client._session is None + + async def test_aclose_closes_odata(self): + """aclose() closes the internal _odata client.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + mock_odata = AsyncMock() + client._odata = mock_odata + await client.aclose() + mock_odata.close.assert_called_once() + assert client._odata is None + + async def test_aclose_idempotent(self): + """aclose() is safe to call multiple times.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + await client.aclose() + await client.aclose() # should not raise + assert client._closed + + async def test_context_manager_closes_on_exit(self): + """Using async with calls aclose() on exit.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session = MagicMock() + mock_session.close = AsyncMock() + mock_session_cls.return_value = mock_session + async with client: + pass + assert client._closed + + +class TestAsyncDataverseClientCheckClosed: + """Tests for _check_closed guard.""" + + def test_check_closed_raises_when_closed(self): + """_check_closed() raises RuntimeError when the client is closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + client._check_closed() + + def test_check_closed_does_not_raise_when_open(self): + """_check_closed() does not raise when the client is open.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._check_closed() # should not raise + + +class TestAsyncDataverseClientGetOdata: + """Tests for _get_odata() lazy initialisation of the internal OData client.""" + + async def test_get_odata_creates_client_on_first_call(self): + """_get_odata() instantiates _AsyncODataClient and stores it in _odata on first call.""" + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._odata is None + od = client._get_odata() + assert isinstance(od, _AsyncODataClient) + assert client._odata is od + await client.aclose() + + async def test_get_odata_returns_same_instance(self): + """Subsequent calls to _get_odata() return the same cached instance.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + od1 = client._get_odata() + od2 = client._get_odata() + assert od1 is od2 + await client.aclose() + + +class TestAsyncDataverseClientScopedOdata: + """Tests for _scoped_odata(), an async context manager that guards OData client access.""" + + async def test_scoped_odata_yields_odata_client(self): + """_scoped_odata() yields the low-level _AsyncODataClient instance.""" + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + async with client._scoped_odata() as od: + assert isinstance(od, _AsyncODataClient) + + async def test_scoped_odata_raises_when_closed(self): + """RuntimeError is raised when _scoped_odata() is entered after the client is closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + async with client._scoped_odata(): + pass + + +class TestAsyncDataverseClientOperationContext: + """Tests for the context= kwarg on AsyncDataverseClient.""" + + def test_context_kwarg_sets_config(self): + """context= stores OperationContext in _config.operation_context.""" + ctx = OperationContext(user_agent_context="app=test/1.0;agent=claude-code") + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential(), context=ctx) + assert client._config.operation_context.user_agent_context == "app=test/1.0;agent=claude-code" + + def test_no_context_leaves_config_default(self): + """Without context=, operation_context defaults to None.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._config.operation_context is None + + def test_config_and_context_raises(self): + """Providing both config= and context= raises ValueError.""" + ctx = OperationContext(user_agent_context="app=test/1.0") + config = DataverseConfig(operation_context=ctx) + with pytest.raises(ValueError, match="config.*context|context.*config"): + AsyncDataverseClient( + "https://org.crm.dynamics.com", + _make_credential(), + config=config, + context=OperationContext(user_agent_context="app=other/2.0"), + ) + + def test_config_alone_works(self): + """Providing config= without context= uses config's operation_context.""" + ctx = OperationContext(user_agent_context="app=test/1.0;skill=dv") + config = DataverseConfig(operation_context=ctx) + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential(), config=config) + assert client._config.operation_context.user_agent_context == "app=test/1.0;skill=dv" diff --git a/tests/unit/aio/test_async_dataframe.py b/tests/unit/aio/test_async_dataframe.py new file mode 100644 index 00000000..36b69760 --- /dev/null +++ b/tests/unit/aio/test_async_dataframe.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +import pandas as pd +from contextlib import asynccontextmanager +from unittest.mock import MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_dataframe import AsyncDataFrameOperations + + +def _make_client_with_od(mock_od): + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped(): + yield mock_od + + client._scoped_odata = _fake_scoped + return client + + +class TestAsyncDataFrameOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.dataframe, AsyncDataFrameOperations) + + +class TestAsyncDataFrameSql: + async def test_sql_returns_dataframe(self, async_client, mock_od): + """sql() executes a SQL query and returns a DataFrame.""" + mock_od._query_sql.return_value = [ + {"name": "Contoso", "accountid": "guid-1"}, + {"name": "Fabrikam", "accountid": "guid-2"}, + ] + + df = await async_client.dataframe.sql("SELECT name FROM account") + + assert isinstance(df, pd.DataFrame) + assert len(df) == 2 + assert "name" in df.columns + + async def test_sql_empty_result_returns_empty_dataframe(self, async_client, mock_od): + """sql() returns an empty DataFrame when no rows match.""" + mock_od._query_sql.return_value = [] + df = await async_client.dataframe.sql("SELECT name FROM account WHERE 1=0") + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + + +class TestAsyncDataFrameCreate: + async def test_create_returns_series_of_guids(self, async_client, mock_od): + """create() returns a Series of GUIDs aligned with the input DataFrame.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1", "guid-2"] + + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + result = await async_client.dataframe.create("account", df) + + assert isinstance(result, pd.Series) + assert list(result) == ["guid-1", "guid-2"] + + async def test_create_non_dataframe_raises(self, async_client, mock_od): + """create() raises TypeError if records is not a DataFrame.""" + with pytest.raises(TypeError): + await async_client.dataframe.create("account", [{"name": "X"}]) + + async def test_create_empty_dataframe_raises(self, async_client, mock_od): + """create() raises ValueError if records is empty.""" + with pytest.raises(ValueError): + await async_client.dataframe.create("account", pd.DataFrame()) + + async def test_create_all_null_row_raises(self, async_client, mock_od): + """create() raises ValueError if any row has no non-null values.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + df = pd.DataFrame([{"name": None}]) + with pytest.raises(ValueError, match="no non-null values"): + await async_client.dataframe.create("account", df) + + async def test_create_id_count_mismatch_raises(self, async_client, mock_od): + """create() raises ValueError if the server returns wrong number of IDs.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1"] # 1 ID for 2 rows + + df = pd.DataFrame([{"name": "A"}, {"name": "B"}]) + with pytest.raises(ValueError, match="returned"): + await async_client.dataframe.create("account", df) + + +class TestAsyncDataFrameUpdate: + async def test_update_single_row(self, async_client, mock_od): + """update() with a single-row DataFrame calls records.update once.""" + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": "555"}]) + await async_client.dataframe.update("account", df, id_column="accountid") + mock_od._update.assert_called_once_with("account", "guid-1", {"telephone1": "555"}) + + async def test_update_multiple_rows(self, async_client, mock_od): + """update() with multiple rows calls records.update with lists.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": "555"}, + {"accountid": "guid-2", "telephone1": "666"}, + ] + ) + await async_client.dataframe.update("account", df, id_column="accountid") + mock_od._update_by_ids.assert_called_once_with( + "account", + ["guid-1", "guid-2"], + [{"telephone1": "555"}, {"telephone1": "666"}], + ) + + async def test_update_non_dataframe_raises(self, async_client, mock_od): + """update() raises TypeError if changes is not a DataFrame.""" + with pytest.raises(TypeError): + await async_client.dataframe.update("account", [{}], id_column="id") + + async def test_update_empty_dataframe_raises(self, async_client, mock_od): + """update() raises ValueError if changes is empty.""" + with pytest.raises(ValueError): + await async_client.dataframe.update("account", pd.DataFrame(), id_column="id") + + async def test_update_missing_id_column_raises(self, async_client, mock_od): + """update() raises ValueError if id_column is not in the DataFrame.""" + df = pd.DataFrame([{"name": "X"}]) + with pytest.raises(ValueError, match="id_column"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_invalid_ids_raises(self, async_client, mock_od): + """update() raises ValueError if id_column contains invalid (non-string) values.""" + df = pd.DataFrame([{"accountid": None, "name": "X"}]) + with pytest.raises(ValueError, match="invalid values"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_no_change_columns_raises(self, async_client, mock_od): + """update() raises ValueError if no columns exist besides id_column.""" + df = pd.DataFrame([{"accountid": "guid-1"}]) + with pytest.raises(ValueError, match="No columns to update"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_all_null_rows_skipped(self, async_client, mock_od): + """update() skips rows where all change values are NaN/None.""" + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": None}]) + await async_client.dataframe.update("account", df, id_column="accountid") + # All values are null -> no updates sent + mock_od._update.assert_not_called() + mock_od._update_by_ids.assert_not_called() + + +class TestAsyncDataFrameDelete: + async def test_delete_single(self, async_client, mock_od): + """delete() with a single-element Series calls records.delete once.""" + ids = pd.Series(["guid-1"]) + result = await async_client.dataframe.delete("account", ids) + mock_od._delete.assert_called_once_with("account", "guid-1") + assert result is None + + async def test_delete_multiple_bulk(self, async_client, mock_od): + """delete() with multiple IDs and use_bulk_delete=True uses BulkDelete.""" + mock_od._delete_multiple.return_value = "job-guid" + ids = pd.Series(["guid-1", "guid-2"]) + result = await async_client.dataframe.delete("account", ids) + mock_od._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2"]) + assert result == "job-guid" + + async def test_delete_empty_series_returns_none(self, async_client, mock_od): + """delete() with an empty Series returns None without calling _delete.""" + result = await async_client.dataframe.delete("account", pd.Series([], dtype=str)) + mock_od._delete.assert_not_called() + assert result is None + + async def test_delete_non_series_raises(self, async_client, mock_od): + """delete() raises TypeError if ids is not a pandas Series.""" + with pytest.raises(TypeError): + await async_client.dataframe.delete("account", ["guid-1", "guid-2"]) + + async def test_delete_invalid_ids_raises(self, async_client, mock_od): + """delete() raises ValueError if any ID is not a non-empty string.""" + ids = pd.Series(["guid-1", None]) + with pytest.raises(ValueError, match="invalid values"): + await async_client.dataframe.delete("account", ids) diff --git a/tests/unit/aio/test_async_files.py b/tests/unit/aio/test_async_files.py new file mode 100644 index 00000000..9c326d07 --- /dev/null +++ b/tests/unit/aio/test_async_files.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations + + +class TestAsyncFileOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.files, AsyncFileOperations) + + +class TestAsyncFileUpload: + async def test_upload_delegates_to_upload_file(self, async_client, mock_od): + """upload() calls od._upload_file with all provided arguments.""" + await async_client.files.upload( + "account", + "guid-1", + "new_Document", + "/path/to/file.pdf", + mode="small", + mime_type="application/pdf", + if_none_match=False, + ) + + mock_od._upload_file.assert_called_once_with( + "account", + "guid-1", + "new_Document", + "/path/to/file.pdf", + mode="small", + mime_type="application/pdf", + if_none_match=False, + ) + + async def test_upload_default_args(self, async_client, mock_od): + """upload() passes None/True for optional args when not specified.""" + await async_client.files.upload("account", "guid-1", "new_Doc", "/path/file.txt") + + mock_od._upload_file.assert_called_once_with( + "account", + "guid-1", + "new_Doc", + "/path/file.txt", + mode=None, + mime_type=None, + if_none_match=True, + ) + + async def test_upload_returns_none(self, async_client, mock_od): + """upload() returns None.""" + result = await async_client.files.upload("account", "guid-1", "new_Doc", "/path/file.txt") + assert result is None diff --git a/tests/unit/aio/test_async_query.py b/tests/unit/aio/test_async_query.py new file mode 100644 index 00000000..3d867bd3 --- /dev/null +++ b/tests/unit/aio/test_async_query.py @@ -0,0 +1,542 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.models.async_fetchxml_query import AsyncFetchXmlQuery +from PowerPlatform.Dataverse.aio.models.async_query_builder import AsyncQueryBuilder +from PowerPlatform.Dataverse.models.record import QueryResult, Record + + +def _make_async_client_with_od(mock_od): + """Helper: create async client with mocked _scoped_odata.""" + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped(): + yield mock_od + + client._scoped_odata = _fake_scoped + return client + + +_SIMPLE_FETCHXML = '' + + +class TestAsyncQueryOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.query, AsyncQueryOperations) + + def test_builder_returns_async_query_builder(self, async_client): + """builder() returns an AsyncQueryBuilder bound to this client.""" + qb = async_client.query.builder("account") + assert isinstance(qb, AsyncQueryBuilder) + assert qb._query_ops is async_client.query + + def test_fetchxml_returns_async_fetchxml_query(self, async_client): + """fetchxml() returns an AsyncFetchXmlQuery for valid XML.""" + q = async_client.query.fetchxml(_SIMPLE_FETCHXML) + assert isinstance(q, AsyncFetchXmlQuery) + assert q._entity_name == "account" + + +class TestAsyncQueryBuilder: + async def test_execute_returns_query_result(self, async_client, mock_od): + """builder().execute() collects all pages into a QueryResult.""" + + async def _pages(*args, **kwargs): + yield [{"name": "Contoso", "accountid": "g1"}] + yield [{"name": "Fabrikam", "accountid": "g2"}] + + mock_od._get_multiple = _pages + + result = await async_client.query.builder("account").select("name").execute() + + assert isinstance(result, QueryResult) + assert len(result) == 2 + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_execute_pages_yields_per_page(self, async_client, mock_od): + """builder().execute_pages() yields one QueryResult per page.""" + + async def _pages(*args, **kwargs): + yield [{"name": "A", "accountid": "g1"}] + yield [{"name": "B", "accountid": "g2"}] + + mock_od._get_multiple = _pages + + pages = [] + async for page in async_client.query.builder("account").select("name").execute_pages(): + pages.append(page) + + assert len(pages) == 2 + assert pages[0][0]["name"] == "A" + assert pages[1][0]["name"] == "B" + + async def test_execute_raises_without_scope(self, async_client): + """execute() raises ValueError when no select/where/top/page_size is set.""" + with pytest.raises(ValueError, match="full-table scans"): + await async_client.query.builder("account").execute() + + async def test_execute_raises_when_unbound(self): + """execute() raises RuntimeError when builder was not created via client.query.builder().""" + qb = AsyncQueryBuilder("account") + qb.select("name") + with pytest.raises(RuntimeError, match="client.query.builder"): + await qb.execute() + + async def test_execute_pages_raises_without_scope(self, async_client): + """execute_pages() raises ValueError when no scope constraint is set.""" + with pytest.raises(ValueError, match="full-table scans"): + async for _ in async_client.query.builder("account").execute_pages(): + pass + + def test_chaining_methods_return_self(self, async_client): + """All fluent methods return the same AsyncQueryBuilder instance.""" + from PowerPlatform.Dataverse.models.filters import col + + qb = async_client.query.builder("account") + assert qb.select("name") is qb + assert qb.where(col("statecode") == 0) is qb + assert qb.order_by("name") is qb + assert qb.top(10) is qb + assert qb.page_size(5) is qb + + +class TestAsyncFetchXmlQueryFactory: + def test_fetchxml_invalid_type_raises(self, async_client): + """fetchxml() raises ValidationError when xml is not a string.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError): + async_client.query.fetchxml(123) + + def test_fetchxml_empty_raises(self, async_client): + """fetchxml() raises ValidationError for empty string.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError): + async_client.query.fetchxml(" ") + + def test_fetchxml_malformed_raises(self, async_client): + """fetchxml() raises ValidationError for malformed XML.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError, match="not well-formed"): + async_client.query.fetchxml("") + + def test_fetchxml_missing_entity_element_raises(self, async_client): + """fetchxml() raises ValueError when element is absent.""" + with pytest.raises(ValueError, match=""): + async_client.query.fetchxml("") + + def test_fetchxml_missing_entity_name_raises(self, async_client): + """fetchxml() raises ValueError when has no name attribute.""" + with pytest.raises(ValueError, match="name"): + async_client.query.fetchxml("") + + +class TestAsyncFetchXmlQueryExecution: + async def test_execute_returns_query_result(self, async_client, mock_od): + """AsyncFetchXmlQuery.execute() collects all pages into a QueryResult.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(return_value=resp) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert isinstance(result, QueryResult) + assert len(result) == 1 + assert result[0]["name"] == "Contoso" + + async def test_execute_pages_yields_pages(self, async_client, mock_od): + """AsyncFetchXmlQuery.execute_pages() yields one QueryResult per page.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(return_value=resp) + + pages = [] + async for page in async_client.query.fetchxml(_SIMPLE_FETCHXML).execute_pages(): + pages.append(page) + + assert len(pages) == 1 + assert pages[0][0]["name"] == "Contoso" + + +class TestAsyncQuerySql: + async def test_sql_returns_records(self, async_client, mock_od): + """sql() calls _query_sql and wraps results in Record objects.""" + mock_od._query_sql.return_value = [ + {"name": "Contoso", "accountid": "guid-1"}, + {"name": "Fabrikam", "accountid": "guid-2"}, + ] + + result = await async_client.query.sql("SELECT TOP 2 name FROM account") + + mock_od._query_sql.assert_called_once_with("SELECT TOP 2 name FROM account") + assert len(result) == 2 + assert all(isinstance(r, Record) for r in result) + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_sql_empty_result(self, async_client, mock_od): + """sql() returns an empty list when no rows match.""" + mock_od._query_sql.return_value = [] + result = await async_client.query.sql("SELECT name FROM account WHERE name = 'X'") + assert result == [] + + +class TestAsyncQuerySqlColumns: + async def test_sql_columns_filters_virtual_and_system(self, async_client, mock_od): + """sql_columns() calls tables.list_columns and filters out virtual/system columns.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "accountid", + "AttributeType": "Uniqueidentifier", + "IsPrimaryId": True, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "versionnumber", + "AttributeType": "BigInt", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + ] + + cols = await async_client.query.sql_columns("account") + + # versionnumber is a system column — excluded by default + names = [c["name"] for c in cols] + assert "versionnumber" not in names + assert "accountid" in names + assert "name" in names + + async def test_sql_columns_include_system(self, async_client, mock_od): + """sql_columns(include_system=True) includes system columns.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "versionnumber", + "AttributeType": "BigInt", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + } + ] + + cols = await async_client.query.sql_columns("account", include_system=True) + assert any(c["name"] == "versionnumber" for c in cols) + + async def test_sql_columns_excludes_attribute_of(self, async_client, mock_od): + """sql_columns() excludes columns where AttributeOf is set.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "parentcustomeridname", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": "parentcustomerid", + } + ] + + cols = await async_client.query.sql_columns("contact") + assert cols == [] + + async def test_sql_columns_skips_empty_logical_name(self, async_client, mock_od): + """sql_columns() skips columns where LogicalName is empty.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + "AttributeOf": None, + }, + ] + cols = await async_client.query.sql_columns("account") + names = [c["name"] for c in cols] + assert "" not in names + assert "name" in names + + async def test_sql_columns_extracts_display_label(self, async_client, mock_od): + """sql_columns() extracts label from UserLocalizedLabel when present.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {"UserLocalizedLabel": {"Label": "Account Name", "LanguageCode": 1033}}, + "AttributeOf": None, + }, + ] + cols = await async_client.query.sql_columns("account") + assert len(cols) == 1 + assert cols[0]["label"] == "Account Name" + + +class TestAsyncQueryOdataExpands: + async def test_odata_expands_returns_nav_properties(self, async_client, mock_od): + """odata_expands() returns navigation property metadata.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.return_value = "accounts" + + result = await async_client.query.odata_expands("contact") + + assert len(result) == 1 + assert result[0]["nav_property"] == "parentcustomerid_account" + assert result[0]["target_table"] == "account" + + async def test_odata_expands_filters_non_referencing(self, async_client, mock_od): + """odata_expands() skips relationships where ReferencingEntity != table.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "account", # not "contact" + "ReferencingEntityNavigationPropertyName": "ownerid_systemuser", + "ReferencedEntity": "systemuser", + "ReferencingAttribute": "ownerid", + "SchemaName": "account_owner_rel", + } + ] + mock_od._entity_set_from_schema_name.return_value = "systemusers" + + result = await async_client.query.odata_expands("contact") + assert result == [] + + async def test_odata_expands_skips_empty_nav_prop(self, async_client, mock_od): + """odata_expands() skips relationships with empty nav_prop or target.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "", # empty nav prop + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.return_value = "accounts" + + result = await async_client.query.odata_expands("contact") + assert result == [] + + async def test_odata_expands_handles_entity_set_resolution_failure(self, async_client, mock_od): + """odata_expands() sets target_entity_set to '' when resolution raises.""" + from PowerPlatform.Dataverse.core.errors import MetadataError + + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.side_effect = MetadataError("not found") + + result = await async_client.query.odata_expands("contact") + + assert len(result) == 1 + assert result[0]["target_entity_set"] == "" + + +class TestAsyncFetchXmlQueryFactoryUrlLength: + def test_fetchxml_url_too_long_raises(self, async_client): + """fetchxml() raises ValidationError when encoded XML exceeds the URL length limit.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + # Build XML long enough to exceed _MAX_URL_LENGTH when encoded + long_xml = '' + '' * 1200 + "" + with pytest.raises(ValidationError, match="URL length limit"): + async_client.query.fetchxml(long_xml) + + +class TestAsyncFetchXmlQueryPaging: + """Tests for multi-page FetchXML execution paths.""" + + async def test_execute_multi_page_with_cookie(self, async_client, mock_od): + """execute() follows paging cookies across multiple pages.""" + import urllib.parse + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + inner = '' + encoded = urllib.parse.quote(urllib.parse.quote(inner)) + paging_cookie = f'' + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": paging_cookie, + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_execute_multi_page_cookie_parse_error_fallback(self, async_client, mock_od): + """execute() falls back to simple paging when the cookie XML is malformed.""" + import warnings + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": "<<>>", + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert any("paging cookie could not be parsed" in str(warning.message) for warning in w) + + async def test_execute_multi_page_no_cookie_simple_paging(self, async_client, mock_od): + """execute() falls back to simple page-number paging when no cookie is returned.""" + import warnings + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + # No fetchxmlpagingcookie key + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert any("simple paging" in str(warning.message) for warning in w) + + async def test_execute_json_parse_error_yields_empty_page(self, async_client, mock_od): + """execute() yields an empty page when the response body cannot be parsed as JSON.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock(side_effect=Exception("invalid json")) + mock_od._request = AsyncMock(return_value=resp) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + assert len(result) == 0 + + async def test_execute_raises_on_max_pages_exceeded(self, async_client, mock_od): + """execute() raises ValidationError when paging exceeds the maximum page limit.""" + import urllib.parse + import warnings + from PowerPlatform.Dataverse.core.errors import ValidationError + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + def _make_page_resp(page_num: int): + inner = f'' + encoded = urllib.parse.quote(urllib.parse.quote(inner)) + cookie = f'' + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": f"Record{page_num}", "accountid": f"g{page_num}"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": cookie, + } + ) + return resp + + # Always return morerecords=True to trigger the limit + mock_od._request = AsyncMock(side_effect=lambda *a, **kw: _make_page_resp(1)) + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + with pytest.raises(ValidationError, match="exceeded"): + await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() diff --git a/tests/unit/aio/test_async_records.py b/tests/unit/aio/test_async_records.py new file mode 100644 index 00000000..13b72582 --- /dev/null +++ b/tests/unit/aio/test_async_records.py @@ -0,0 +1,541 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import warnings +import pytest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.upsert import UpsertItem + +# --------------------------------------------------------------------------- +# Async generator helpers used by list/list_pages tests +# --------------------------------------------------------------------------- + + +async def _agen(*pages): + """Yield each argument as one page from an async generator.""" + for p in pages: + yield p + + +class TestAsyncRecordOperationsNamespace: + """Verify the namespace attribute type.""" + + def test_namespace_type(self, async_client): + assert isinstance(async_client.records, AsyncRecordOperations) + + +class TestAsyncRecordCreate: + """Tests for AsyncRecordOperations.create.""" + + async def test_create_single(self, async_client, mock_od): + """create() with a single dict calls _entity_set_from_schema_name and _create.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create.return_value = "guid-123" + + result = await async_client.records.create("account", {"name": "Contoso"}) + + mock_od._entity_set_from_schema_name.assert_called_once_with("account") + mock_od._create.assert_called_once_with("accounts", "account", {"name": "Contoso"}) + assert result == "guid-123" + assert isinstance(result, str) + + async def test_create_bulk(self, async_client, mock_od): + """create() with a list of dicts calls _create_multiple.""" + payloads = [{"name": "A"}, {"name": "B"}] + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1", "guid-2"] + + result = await async_client.records.create("account", payloads) + + mock_od._create_multiple.assert_called_once_with("accounts", "account", payloads) + assert result == ["guid-1", "guid-2"] + + async def test_create_single_non_string_return_raises(self, async_client, mock_od): + """create() raises TypeError if _create returns a non-string.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create.return_value = 12345 + + with pytest.raises(TypeError): + await async_client.records.create("account", {"name": "X"}) + + async def test_create_bulk_non_list_return_raises(self, async_client, mock_od): + """create() raises TypeError if _create_multiple returns a non-list.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = "not-a-list" + + with pytest.raises(TypeError): + await async_client.records.create("account", [{"name": "X"}]) + + async def test_create_invalid_data_type_raises(self, async_client, mock_od): + """create() raises TypeError if data is neither dict nor list.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + with pytest.raises(TypeError): + await async_client.records.create("account", "invalid") + + +class TestAsyncRecordUpdate: + """Tests for AsyncRecordOperations.update.""" + + async def test_update_single(self, async_client, mock_od): + """update() with a str id and dict changes calls _update.""" + await async_client.records.update("account", "guid-1", {"telephone1": "555"}) + mock_od._update.assert_called_once_with("account", "guid-1", {"telephone1": "555"}) + + async def test_update_broadcast(self, async_client, mock_od): + """update() with list of ids and single dict calls _update_by_ids.""" + await async_client.records.update("account", ["id-1", "id-2"], {"statecode": 1}) + mock_od._update_by_ids.assert_called_once_with("account", ["id-1", "id-2"], {"statecode": 1}) + + async def test_update_paired(self, async_client, mock_od): + """update() with list of ids and list of dicts calls _update_by_ids.""" + await async_client.records.update("account", ["id-1", "id-2"], [{"name": "A"}, {"name": "B"}]) + mock_od._update_by_ids.assert_called_once_with("account", ["id-1", "id-2"], [{"name": "A"}, {"name": "B"}]) + + async def test_update_single_non_dict_changes_raises(self, async_client, mock_od): + """update() raises TypeError if ids is str but changes is not a dict.""" + with pytest.raises(TypeError): + await async_client.records.update("account", "guid-1", ["not", "a", "dict"]) + + async def test_update_invalid_ids_type_raises(self, async_client, mock_od): + """update() raises TypeError if ids is neither str nor list.""" + with pytest.raises(TypeError): + await async_client.records.update("account", 12345, {"name": "X"}) + + async def test_update_returns_none(self, async_client, mock_od): + """update() returns None.""" + result = await async_client.records.update("account", "guid-1", {"name": "X"}) + assert result is None + + +class TestAsyncRecordDelete: + """Tests for AsyncRecordOperations.delete.""" + + async def test_delete_single(self, async_client, mock_od): + """delete() with a str id calls _delete and returns None.""" + result = await async_client.records.delete("account", "guid-to-delete") + mock_od._delete.assert_called_once_with("account", "guid-to-delete") + assert result is None + + async def test_delete_bulk(self, async_client, mock_od): + """delete() with a list of ids uses _delete_multiple by default.""" + mock_od._delete_multiple.return_value = "job-guid-456" + result = await async_client.records.delete("account", ["id-1", "id-2", "id-3"]) + mock_od._delete_multiple.assert_called_once_with("account", ["id-1", "id-2", "id-3"]) + assert result == "job-guid-456" + + async def test_delete_bulk_sequential(self, async_client, mock_od): + """delete() with use_bulk_delete=False calls _delete once per id.""" + result = await async_client.records.delete("account", ["id-1", "id-2"], use_bulk_delete=False) + assert mock_od._delete.call_count == 2 + mock_od._delete.assert_any_call("account", "id-1") + mock_od._delete.assert_any_call("account", "id-2") + mock_od._delete_multiple.assert_not_called() + assert result is None + + async def test_delete_empty_list(self, async_client, mock_od): + """delete() with an empty list returns None without calling _delete.""" + result = await async_client.records.delete("account", []) + mock_od._delete.assert_not_called() + mock_od._delete_multiple.assert_not_called() + assert result is None + + async def test_delete_invalid_ids_type_raises(self, async_client, mock_od): + """delete() raises TypeError if ids is neither str nor list.""" + with pytest.raises(TypeError): + await async_client.records.delete("account", 12345) + + async def test_delete_list_with_non_string_guid_raises(self, async_client, mock_od): + """delete() raises TypeError if the ids list contains non-string entries.""" + with pytest.raises(TypeError): + await async_client.records.delete("account", ["valid-guid", 42]) + + +class TestAsyncRecordUpsert: + """Tests for AsyncRecordOperations.upsert.""" + + async def test_upsert_single_upsert_item(self, async_client, mock_od): + """upsert() with a single UpsertItem calls _upsert.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) + + result = await async_client.records.upsert("account", [item]) + + mock_od._upsert.assert_called_once_with( + "accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"} + ) + mock_od._upsert_multiple.assert_not_called() + assert result is None + + async def test_upsert_single_dict(self, async_client, mock_od): + """upsert() with a single dict item calls _upsert.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + item = {"alternate_key": {"accountnumber": "ACC-001"}, "record": {"name": "Contoso"}} + + await async_client.records.upsert("account", [item]) + + mock_od._upsert.assert_called_once_with( + "accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"} + ) + + async def test_upsert_multiple_calls_upsert_multiple(self, async_client, mock_od): + """upsert() with multiple items calls _upsert_multiple.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + items = [ + UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "Contoso"}), + UpsertItem(alternate_key={"accountnumber": "B"}, record={"name": "Fabrikam"}), + ] + + await async_client.records.upsert("account", items) + + mock_od._upsert_multiple.assert_called_once_with( + "accounts", + "account", + [{"accountnumber": "A"}, {"accountnumber": "B"}], + [{"name": "Contoso"}, {"name": "Fabrikam"}], + ) + mock_od._upsert.assert_not_called() + + async def test_upsert_empty_list_raises(self, async_client, mock_od): + """upsert() with an empty list raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", []) + + async def test_upsert_non_list_raises(self, async_client, mock_od): + """upsert() with a non-list argument raises TypeError.""" + item = UpsertItem(alternate_key={"accountnumber": "X"}, record={"name": "Y"}) + with pytest.raises(TypeError): + await async_client.records.upsert("account", item) + + async def test_upsert_invalid_item_raises(self, async_client, mock_od): + """upsert() with an item that is neither UpsertItem nor valid dict raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", [42]) + + async def test_upsert_dict_missing_record_key_raises(self, async_client, mock_od): + """upsert() with a dict missing the 'record' key raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", [{"alternate_key": {"name": "acc1"}}]) + + +# --------------------------------------------------------------------------- +# retrieve() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordRetrieve: + """Tests for AsyncRecordOperations.retrieve().""" + + async def test_retrieve_returns_record(self, async_client, mock_od): + """retrieve() returns a Record instance.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + result = await async_client.records.retrieve("account", "abc") + assert isinstance(result, Record) + assert result["name"] == "Contoso" + + async def test_retrieve_passes_select(self, async_client, mock_od): + """retrieve() passes select= to _get.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + await async_client.records.retrieve("account", "abc", select=["name"]) + mock_od._get.assert_called_once_with("account", "abc", select=["name"], expand=None, include_annotations=None) + + async def test_retrieve_passes_expand(self, async_client, mock_od): + """retrieve() passes expand= to _get.""" + mock_od._get.return_value = { + "accountid": "abc", + "primarycontactid": {"contactid": "cid", "fullname": "John Doe"}, + } + result = await async_client.records.retrieve("account", "abc", expand=["primarycontactid"]) + mock_od._get.assert_called_once_with( + "account", "abc", select=None, expand=["primarycontactid"], include_annotations=None + ) + assert result["primarycontactid"]["fullname"] == "John Doe" + + async def test_retrieve_passes_select_and_expand(self, async_client, mock_od): + """retrieve() passes both select= and expand= to _get.""" + mock_od._get.return_value = {"name": "Contoso", "primarycontactid": {"fullname": "John"}} + await async_client.records.retrieve("account", "abc", select=["name"], expand=["primarycontactid"]) + mock_od._get.assert_called_once_with( + "account", "abc", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + + async def test_retrieve_passes_include_annotations(self, async_client, mock_od): + """retrieve() passes include_annotations= to _get.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get.return_value = { + "accountid": "abc", + "statuscode": 1, + f"statuscode@{annotation}": "Active", + } + result = await async_client.records.retrieve("account", "abc", include_annotations=annotation) + mock_od._get.assert_called_once_with("account", "abc", select=None, expand=None, include_annotations=annotation) + assert result[f"statuscode@{annotation}"] == "Active" + + async def test_retrieve_no_deprecation_warning(self, async_client, mock_od): + """retrieve() does not emit DeprecationWarning.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await async_client.records.retrieve("account", "abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"retrieve() must not emit DeprecationWarning: {dep}" + + async def test_retrieve_returns_none_on_404(self, async_client, mock_od): + """retrieve() returns None when _get raises HttpError with status 404.""" + mock_od._get.side_effect = HttpError("Not Found", 404) + result = await async_client.records.retrieve("account", "nonexistent") + assert result is None + + async def test_retrieve_reraises_non_404(self, async_client, mock_od): + """retrieve() re-raises HttpError for non-404 status codes.""" + mock_od._get.side_effect = HttpError("Server Error", 500) + with pytest.raises(HttpError): + await async_client.records.retrieve("account", "some-id") + + async def test_retrieve_reraises_non_http_errors(self, async_client, mock_od): + """retrieve() re-raises non-HttpError exceptions unchanged.""" + mock_od._get.side_effect = ValueError("Bad input") + with pytest.raises(ValueError): + await async_client.records.retrieve("account", "some-id") + + async def test_retrieve_record_id_set(self, async_client, mock_od): + """retrieve() sets record.id from the record_id argument.""" + mock_od._get.return_value = {"name": "Contoso"} + record = await async_client.records.retrieve("account", "my-id") + assert record.id == "my-id" + + async def test_retrieve_table_set(self, async_client, mock_od): + """retrieve() sets record.table from the table argument.""" + mock_od._get.return_value = {"name": "Contoso"} + record = await async_client.records.retrieve("account", "my-id") + assert record.table == "account" + + +# --------------------------------------------------------------------------- +# list() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordList: + """Tests for AsyncRecordOperations.list().""" + + async def test_list_returns_query_result(self, async_client, mock_od): + """list() returns a QueryResult.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + result = await async_client.records.list("account") + assert isinstance(result, QueryResult) + + async def test_list_collects_all_pages(self, async_client, mock_od): + """list() collects records from all pages into one QueryResult.""" + mock_od._get_multiple = MagicMock( + return_value=_agen( + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ) + ) + result = await async_client.records.list("account") + assert len(result) == 3 + + async def test_list_no_deprecation_warning(self, async_client, mock_od): + """list() does not emit DeprecationWarning.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await async_client.records.list("account", filter="statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"list() must not emit DeprecationWarning: {dep}" + + async def test_list_passes_string_filter(self, async_client, mock_od): + """list() passes a string filter to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", filter="statecode eq 0") + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_passes_filter_expression(self, async_client, mock_od): + """list() converts a FilterExpression to string before passing to _get_multiple.""" + from PowerPlatform.Dataverse.models.filters import col + + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", filter=col("statecode") == 0) + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_passes_select(self, async_client, mock_od): + """list() passes select= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", select=["name", "revenue"]) + assert mock_od._get_multiple.call_args[1]["select"] == ["name", "revenue"] + + async def test_list_passes_top(self, async_client, mock_od): + """list() passes top= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", top=50) + assert mock_od._get_multiple.call_args[1]["top"] == 50 + + async def test_list_none_filter_passes_none(self, async_client, mock_od): + """list() passes filter=None to _get_multiple when no filter specified.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account") + assert mock_od._get_multiple.call_args[1]["filter"] is None + + async def test_list_result_iterable(self, async_client, mock_od): + """list() result is iterable and contains Record instances.""" + mock_od._get_multiple = MagicMock(return_value=_agen([{"name": "X", "accountid": "1"}])) + result = await async_client.records.list("account") + records = list(result) + assert len(records) == 1 + assert records[0]["name"] == "X" + + async def test_list_result_to_dataframe(self, async_client, mock_od): + """list() result can be converted to a DataFrame.""" + import pandas as pd + + mock_od._get_multiple = MagicMock( + return_value=_agen([{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}]) + ) + df = (await async_client.records.list("account", select=["name"])).to_dataframe() + assert isinstance(df, pd.DataFrame) + assert len(df) == 2 + + async def test_list_passes_orderby(self, async_client, mock_od): + """list() passes orderby= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", orderby=["name asc"]) + assert mock_od._get_multiple.call_args[1]["orderby"] == ["name asc"] + + async def test_list_passes_expand(self, async_client, mock_od): + """list() passes expand= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", expand=["primarycontactid"]) + assert mock_od._get_multiple.call_args[1]["expand"] == ["primarycontactid"] + + async def test_list_passes_page_size(self, async_client, mock_od): + """list() passes page_size= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", page_size=200) + assert mock_od._get_multiple.call_args[1]["page_size"] == 200 + + async def test_list_passes_count(self, async_client, mock_od): + """list() passes count=True to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", count=True) + assert mock_od._get_multiple.call_args[1]["count"] is True + + async def test_list_passes_include_annotations(self, async_client, mock_od): + """list() passes include_annotations= to _get_multiple.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", include_annotations=annotation) + assert mock_od._get_multiple.call_args[1]["include_annotations"] == annotation + + +# --------------------------------------------------------------------------- +# list_pages() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordListPages: + """Tests for AsyncRecordOperations.list_pages().""" + + async def test_list_pages_is_async_generator(self, async_client, mock_od): + """list_pages() returns an async generator.""" + import inspect + + mock_od._get_multiple = MagicMock(return_value=_agen()) + result = async_client.records.list_pages("account") + assert inspect.isasyncgen(result) + + async def test_list_pages_yields_query_result_per_page(self, async_client, mock_od): + """list_pages() yields one QueryResult per HTTP page.""" + mock_od._get_multiple = MagicMock( + return_value=_agen([{"name": "A", "accountid": "1"}], [{"name": "B", "accountid": "2"}]) + ) + pages = [] + async for page in async_client.records.list_pages("account"): + pages.append(page) + assert len(pages) == 2 + for page in pages: + assert isinstance(page, QueryResult) + + async def test_list_pages_page_contents(self, async_client, mock_od): + """list_pages() preserves per-page record counts.""" + mock_od._get_multiple = MagicMock( + return_value=_agen( + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ) + ) + pages = [] + async for page in async_client.records.list_pages("account"): + pages.append(page) + assert len(pages[0]) == 1 + assert len(pages[1]) == 2 + + async def test_list_pages_no_deprecation_warning(self, async_client, mock_od): + """list_pages() does not emit DeprecationWarning.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + async for _ in async_client.records.list_pages("account", filter="statecode eq 0"): + pass + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"list_pages() must not emit DeprecationWarning: {dep}" + + async def test_list_pages_passes_filter(self, async_client, mock_od): + """list_pages() passes filter= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", filter="statecode eq 0"): + pass + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_pages_passes_select(self, async_client, mock_od): + """list_pages() passes select= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", select=["name"]): + pass + assert mock_od._get_multiple.call_args[1]["select"] == ["name"] + + async def test_list_pages_passes_top(self, async_client, mock_od): + """list_pages() passes top= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", top=50): + pass + assert mock_od._get_multiple.call_args[1]["top"] == 50 + + async def test_list_pages_passes_orderby(self, async_client, mock_od): + """list_pages() passes orderby= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", orderby=["name asc"]): + pass + assert mock_od._get_multiple.call_args[1]["orderby"] == ["name asc"] + + async def test_list_pages_passes_expand(self, async_client, mock_od): + """list_pages() passes expand= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", expand=["primarycontactid"]): + pass + assert mock_od._get_multiple.call_args[1]["expand"] == ["primarycontactid"] + + async def test_list_pages_passes_page_size(self, async_client, mock_od): + """list_pages() passes page_size= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", page_size=200): + pass + assert mock_od._get_multiple.call_args[1]["page_size"] == 200 + + async def test_list_pages_passes_count(self, async_client, mock_od): + """list_pages() passes count=True to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", count=True): + pass + assert mock_od._get_multiple.call_args[1]["count"] is True + + async def test_list_pages_passes_include_annotations(self, async_client, mock_od): + """list_pages() passes include_annotations= to _get_multiple.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", include_annotations=annotation): + pass + assert mock_od._get_multiple.call_args[1]["include_annotations"] == annotation diff --git a/tests/unit/aio/test_async_tables.py b/tests/unit/aio/test_async_tables.py new file mode 100644 index 00000000..366f0c74 --- /dev/null +++ b/tests/unit/aio/test_async_tables.py @@ -0,0 +1,314 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.models.relationship import RelationshipInfo +from PowerPlatform.Dataverse.models.table_info import AlternateKeyInfo, TableInfo +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + + +def _label(text: str = "Test") -> Label: + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +def _table_raw(schema_name: str = "new_Product") -> dict: + return { + "table_schema_name": schema_name, + "entity_set_name": "new_products", + "table_logical_name": "new_product", + "metadata_id": "meta-guid-1", + "columns_created": ["new_Price"], + } + + +def _rel_one_to_many_raw() -> dict: + return { + "relationship_id": "rel-guid-1", + "relationship_schema_name": "new_Dept_Emp", + "lookup_schema_name": "new_DeptId", + "referenced_entity": "new_dept", + "referencing_entity": "new_employee", + } + + +def _rel_many_to_many_raw() -> dict: + return { + "relationship_id": "rel-guid-2", + "relationship_schema_name": "new_emp_proj", + "entity1_logical_name": "new_employee", + "entity2_logical_name": "new_project", + } + + +class TestAsyncTableOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.tables, AsyncTableOperations) + + +class TestAsyncTableCreate: + async def test_create_returns_table_info(self, async_client, mock_od): + """create() returns a TableInfo built from the raw dict.""" + mock_od._create_table.return_value = _table_raw() + columns = {"new_Price": "decimal"} + + result = await async_client.tables.create( + "new_Product", + columns, + solution="MySol", + primary_column="new_ProductName", + display_name="Product", + ) + + mock_od._create_table.assert_called_once_with("new_Product", columns, "MySol", "new_ProductName", "Product") + assert isinstance(result, TableInfo) + assert result.schema_name == "new_Product" + + async def test_create_with_minimal_args(self, async_client, mock_od): + """create() works with only table and columns.""" + mock_od._create_table.return_value = _table_raw() + await async_client.tables.create("new_Product", {}) + mock_od._create_table.assert_called_once_with("new_Product", {}, None, None, None) + + +class TestAsyncTableDelete: + async def test_delete_calls_delete_table(self, async_client, mock_od): + """delete() calls _delete_table with the table schema name.""" + await async_client.tables.delete("new_Product") + mock_od._delete_table.assert_called_once_with("new_Product") + + +class TestAsyncTableGet: + async def test_get_returns_table_info(self, async_client, mock_od): + """get() returns TableInfo when table exists.""" + mock_od._get_table_info.return_value = _table_raw() + result = await async_client.tables.get("new_Product") + assert isinstance(result, TableInfo) + assert result.schema_name == "new_Product" + + async def test_get_returns_none_when_not_found(self, async_client, mock_od): + """get() returns None when _get_table_info returns None.""" + mock_od._get_table_info.return_value = None + result = await async_client.tables.get("new_Product") + assert result is None + + +class TestAsyncTableList: + async def test_list_calls_list_tables(self, async_client, mock_od): + """list() calls _list_tables and returns its result.""" + mock_od._list_tables.return_value = [{"LogicalName": "account"}] + result = await async_client.tables.list() + mock_od._list_tables.assert_called_once_with(filter=None, select=None) + assert result == [{"LogicalName": "account"}] + + async def test_list_with_params(self, async_client, mock_od): + """list() passes filter and select to _list_tables.""" + mock_od._list_tables.return_value = [] + await async_client.tables.list(filter="IsPrivate eq false", select=["LogicalName"]) + mock_od._list_tables.assert_called_once_with(filter="IsPrivate eq false", select=["LogicalName"]) + + +class TestAsyncTableAddColumns: + async def test_add_columns_calls_create_columns(self, async_client, mock_od): + """add_columns() calls _create_columns and returns the result.""" + mock_od._create_columns.return_value = ["new_Notes"] + result = await async_client.tables.add_columns("new_Product", {"new_Notes": "string"}) + mock_od._create_columns.assert_called_once_with("new_Product", {"new_Notes": "string"}) + assert result == ["new_Notes"] + + +class TestAsyncTableRemoveColumns: + async def test_remove_columns_calls_delete_columns(self, async_client, mock_od): + """remove_columns() calls _delete_columns and returns the result.""" + mock_od._delete_columns.return_value = ["new_Notes"] + result = await async_client.tables.remove_columns("new_Product", "new_Notes") + mock_od._delete_columns.assert_called_once_with("new_Product", "new_Notes") + assert result == ["new_Notes"] + + +class TestAsyncTableOneToManyRelationship: + async def test_create_one_to_many(self, async_client, mock_od): + """create_one_to_many_relationship() calls _create_one_to_many_relationship and returns RelationshipInfo.""" + mock_od._create_one_to_many_relationship.return_value = _rel_one_to_many_raw() + + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Department")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + + result = await async_client.tables.create_one_to_many_relationship(lookup, relationship) + + mock_od._create_one_to_many_relationship.assert_called_once_with(lookup, relationship, None) + assert isinstance(result, RelationshipInfo) + assert result.relationship_schema_name == "new_Dept_Emp" + + +class TestAsyncTableManyToManyRelationship: + async def test_create_many_to_many(self, async_client, mock_od): + """create_many_to_many_relationship() calls _create_many_to_many_relationship and returns RelationshipInfo.""" + mock_od._create_many_to_many_relationship.return_value = _rel_many_to_many_raw() + + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + + result = await async_client.tables.create_many_to_many_relationship(relationship) + + mock_od._create_many_to_many_relationship.assert_called_once_with(relationship, None) + assert isinstance(result, RelationshipInfo) + assert result.relationship_schema_name == "new_emp_proj" + + +class TestAsyncTableDeleteRelationship: + async def test_delete_relationship(self, async_client, mock_od): + """delete_relationship() calls _delete_relationship with the relationship_id.""" + await async_client.tables.delete_relationship("rel-guid-1") + mock_od._delete_relationship.assert_called_once_with("rel-guid-1") + + +class TestAsyncTableGetRelationship: + async def test_get_relationship_found(self, async_client, mock_od): + """get_relationship() returns RelationshipInfo when found.""" + raw = { + "@odata.type": "#Microsoft.Dynamics.CRM.OneToManyRelationshipMetadata", + "RelationshipId": "rel-guid-1", + "SchemaName": "new_Dept_Emp", + "RelationshipType": "OneToManyRelationship", + "ReferencedEntity": "new_dept", + "ReferencingEntity": "new_employee", + "ReferencingAttribute": "new_deptid", + } + mock_od._get_relationship.return_value = raw + result = await async_client.tables.get_relationship("new_Dept_Emp") + assert isinstance(result, RelationshipInfo) + + async def test_get_relationship_not_found(self, async_client, mock_od): + """get_relationship() returns None when _get_relationship returns None.""" + mock_od._get_relationship.return_value = None + result = await async_client.tables.get_relationship("nonexistent") + assert result is None + + +class TestAsyncTableCreateLookupField: + async def test_create_lookup_field_builds_models_and_delegates(self, async_client, mock_od): + """create_lookup_field() builds lookup/relationship models and calls create_one_to_many_relationship.""" + from unittest.mock import MagicMock + + mock_lookup = LookupAttributeMetadata(schema_name="new_AccountId", display_name=_label("Account")) + mock_rel = OneToManyRelationshipMetadata( + schema_name="new_account_order", + referenced_entity="account", + referencing_entity="new_order", + referenced_attribute="accountid", + ) + # _build_lookup_field_models is a sync method on _ODataBase; use MagicMock so + # od._build_lookup_field_models(...) returns the tuple directly (not a coroutine). + mock_od._build_lookup_field_models = MagicMock(return_value=(mock_lookup, mock_rel)) + mock_od._create_one_to_many_relationship.return_value = { + "relationship_id": "r-guid", + "relationship_schema_name": "new_account_order", + "lookup_schema_name": "new_AccountId", + "referenced_entity": "account", + "referencing_entity": "new_order", + } + + result = await async_client.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + ) + + mock_od._build_lookup_field_models.assert_called_once() + mock_od._create_one_to_many_relationship.assert_called_once_with(mock_lookup, mock_rel, None) + assert isinstance(result, RelationshipInfo) + + +class TestAsyncTableAlternateKeys: + async def test_create_alternate_key(self, async_client, mock_od): + """create_alternate_key() calls _create_alternate_key and returns AlternateKeyInfo.""" + mock_od._create_alternate_key.return_value = { + "metadata_id": "key-guid", + "schema_name": "new_prod_key", + "key_attributes": ["new_productcode"], + } + + result = await async_client.tables.create_alternate_key( + "new_Product", + "new_prod_key", + ["new_productcode"], + display_name="Product Code", + ) + + mock_od._create_alternate_key.assert_called_once() + assert isinstance(result, AlternateKeyInfo) + assert result.schema_name == "new_prod_key" + assert result.status == "Pending" + + async def test_get_alternate_keys(self, async_client, mock_od): + """get_alternate_keys() calls _get_alternate_keys and returns list of AlternateKeyInfo.""" + mock_od._get_alternate_keys.return_value = [ + { + "MetadataId": "key-guid-1", + "SchemaName": "new_prod_key", + "KeyAttributes": ["new_productcode"], + "EntityKeyIndexStatus": "Active", + } + ] + + result = await async_client.tables.get_alternate_keys("new_Product") + + mock_od._get_alternate_keys.assert_called_once_with("new_Product") + assert len(result) == 1 + assert isinstance(result[0], AlternateKeyInfo) + + async def test_delete_alternate_key(self, async_client, mock_od): + """delete_alternate_key() calls _delete_alternate_key with table and key_id.""" + await async_client.tables.delete_alternate_key("new_Product", "key-guid") + mock_od._delete_alternate_key.assert_called_once_with("new_Product", "key-guid") + + +class TestAsyncTableListColumns: + async def test_list_columns(self, async_client, mock_od): + """list_columns() calls _list_columns and returns its result.""" + mock_od._list_columns.return_value = [{"LogicalName": "name"}] + result = await async_client.tables.list_columns("account") + mock_od._list_columns.assert_called_once_with("account", select=None, filter=None) + assert result == [{"LogicalName": "name"}] + + async def test_list_columns_with_params(self, async_client, mock_od): + """list_columns() passes select and filter to _list_columns.""" + mock_od._list_columns.return_value = [] + await async_client.tables.list_columns( + "account", + select=["LogicalName"], + filter="AttributeType eq 'String'", + ) + mock_od._list_columns.assert_called_once_with( + "account", select=["LogicalName"], filter="AttributeType eq 'String'" + ) + + +class TestAsyncTableListRelationships: + async def test_list_relationships(self, async_client, mock_od): + """list_relationships() calls _list_relationships and returns its result.""" + mock_od._list_relationships.return_value = [{"SchemaName": "new_Dept_Emp"}] + result = await async_client.tables.list_relationships() + mock_od._list_relationships.assert_called_once_with(filter=None, select=None) + assert result == [{"SchemaName": "new_Dept_Emp"}] + + async def test_list_table_relationships(self, async_client, mock_od): + """list_table_relationships() calls _list_table_relationships and returns its result.""" + mock_od._list_table_relationships.return_value = [{"SchemaName": "new_Dept_Emp"}] + result = await async_client.tables.list_table_relationships("account") + mock_od._list_table_relationships.assert_called_once_with("account", filter=None, select=None) + assert result == [{"SchemaName": "new_Dept_Emp"}] diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 39373e05..a0e3f907 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -3,8 +3,6 @@ import pytest from azure.core.credentials import TokenCredential -from PowerPlatform.Dataverse.client import DataverseClient -from PowerPlatform.Dataverse.core.config import DataverseConfig from PowerPlatform.Dataverse.core.errors import HttpError from PowerPlatform.Dataverse.core._error_codes import HTTP_404, HTTP_429, HTTP_500 from PowerPlatform.Dataverse.data._odata import _ODataClient diff --git a/tests/unit/data/test_sql_parse.py b/tests/unit/data/test_sql_parse.py index d01ecd00..e95888df 100644 --- a/tests/unit/data/test_sql_parse.py +++ b/tests/unit/data/test_sql_parse.py @@ -513,7 +513,7 @@ def test_extract_pagingcookie_malformed_url_returns_none(): def test_extract_pagingcookie_exception_returns_none(): """Returns None when an unexpected exception is raised during URL parsing (except branch).""" - with patch("PowerPlatform.Dataverse.data._odata.urlparse", side_effect=RuntimeError("boom")): + with patch("PowerPlatform.Dataverse.data._odata_base.urlparse", side_effect=RuntimeError("boom")): assert _extract_pagingcookie("https://org.example/?$skiptoken=x") is None diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index f40d1e5d..e47cb9a6 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -891,7 +891,6 @@ def test_to_dataframe_forwards_count_and_annotations(self): def test_to_dataframe_with_record_objects(self): """to_dataframe() handles Record objects (with .data attribute).""" import pandas as pd - from PowerPlatform.Dataverse.models.record import Record mock_query_ops, _ = self._make_od( [ diff --git a/tests/unit/test_phase1_ga.py b/tests/unit/test_phase1_ga.py index ee832029..8069ffd1 100644 --- a/tests/unit/test_phase1_ga.py +++ b/tests/unit/test_phase1_ga.py @@ -391,8 +391,6 @@ def test_dataframe_get_warning_message(self): def test_dataframe_other_methods_no_warning(self): """dataframe.sql(), dataframe.create(), etc. must NOT warn.""" - import pandas as pd - from PowerPlatform.Dataverse.models.record import Record self.client._odata._query_sql.return_value = [] diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py index f9e53f9f..6802bc0a 100644 --- a/tests/unit/test_phase3_ga.py +++ b/tests/unit/test_phase3_ga.py @@ -18,7 +18,7 @@ import unittest import warnings from dataclasses import dataclass -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from azure.core.credentials import TokenCredential