From c56d6705f78529f7b8e0b7990b6bfa940e734245 Mon Sep 17 00:00:00 2001 From: Carlos Robles Date: Mon, 21 Sep 2026 22:18:54 -0600 Subject: [PATCH 1/3] Experiment: four tool-specific plugins sharing one skills tree by symlink --- .claude-plugin/marketplace.json | 28 ++ .../.claude-plugin/plugin.json | 23 ++ .../.codex-plugin/plugin.json | 30 ++ .../.cursor-plugin/plugin.json | 25 ++ plugins/microsoft-sql-fdh/plugin.json | 22 ++ .../skills/deploy-app-to-azure | 1 + .../skills/github-actions-for-sql | 1 + .../skills/provision-azure-sql-db | 1 + .../skills/provision-hyperscale | 1 + .../skills/schema-migrations-safely | 1 + .../skills/sql-database-projects | 1 + .../.claude-plugin/plugin.json | 23 ++ .../.codex-plugin/plugin.json | 30 ++ .../.cursor-plugin/plugin.json | 25 ++ plugins/microsoft-sql-migration/plugin.json | 22 ++ .../skills/azuresql-db-from-sql-server | 1 + .../skills/azuresql-db-local-to-cloud | 1 + .../skills/bulk-load-and-bulk-copy | 1 + .../skills/sqlpackage-import-export | 1 + .../.claude-plugin/plugin.json | 23 ++ .../.codex-plugin/plugin.json | 30 ++ .../.cursor-plugin/plugin.json | 25 ++ plugins/microsoft-sql-ssms/plugin.json | 22 ++ .../skills/capture-with-extended-events | 1 + .../skills/diagnose-blocking-and-deadlocks | 1 + .../skills/diagnose-resource-pressure | 1 + .../skills/diagnose-slow-query | 1 + .../microsoft-sql-ssms/skills/entra-id-auth | 1 + .../skills/read-execution-plan | 1 + .../skills/restore-and-recover | 1 + .../skills/rls-multi-tenant | 1 + .../.claude-plugin/plugin.json | 23 ++ .../.codex-plugin/plugin.json | 30 ++ .../.cursor-plugin/plugin.json | 25 ++ plugins/microsoft-sql-vscode/plugin.json | 22 ++ .../skills/azure-functions-sql-bindings | 1 + .../skills/build-app-on-azure-sql | 1 + .../skills/connect-from-dotnet | 1 + .../skills/connect-from-python | 1 + .../skills/connect-from-typescript-and-node | 1 + .../skills/connect-to-azure-sql | 1 + .../skills/design-azure-sql-schema | 1 + .../skills/diagnose-connection-errors | 1 + .../skills/ef-core-azure-sql | 1 + .../skills/embeddings-and-external-models | 1 + .../microsoft-sql-vscode/skills/entra-id-auth | 1 + .../langchain-and-llamaindex-on-azure-sql | 1 + .../skills/prevent-sql-injection | 1 + .../skills/prisma-azure-sql | 1 + .../skills/rag-on-azure-sql | 1 + .../skills/sqlalchemy-azure-sql | 1 + .../skills/t-sql-correctness | 1 + .../skills/t-sql-json-and-openjson | 1 + .../skills/t-sql-upserts-merge | 1 + .../skills/vector-search-azure-sql | 1 + scripts/generate.mjs | 269 +++++++++++++++++- 56 files changed, 731 insertions(+), 4 deletions(-) create mode 100644 plugins/microsoft-sql-fdh/.claude-plugin/plugin.json create mode 100644 plugins/microsoft-sql-fdh/.codex-plugin/plugin.json create mode 100644 plugins/microsoft-sql-fdh/.cursor-plugin/plugin.json create mode 100644 plugins/microsoft-sql-fdh/plugin.json create mode 120000 plugins/microsoft-sql-fdh/skills/deploy-app-to-azure create mode 120000 plugins/microsoft-sql-fdh/skills/github-actions-for-sql create mode 120000 plugins/microsoft-sql-fdh/skills/provision-azure-sql-db create mode 120000 plugins/microsoft-sql-fdh/skills/provision-hyperscale create mode 120000 plugins/microsoft-sql-fdh/skills/schema-migrations-safely create mode 120000 plugins/microsoft-sql-fdh/skills/sql-database-projects create mode 100644 plugins/microsoft-sql-migration/.claude-plugin/plugin.json create mode 100644 plugins/microsoft-sql-migration/.codex-plugin/plugin.json create mode 100644 plugins/microsoft-sql-migration/.cursor-plugin/plugin.json create mode 100644 plugins/microsoft-sql-migration/plugin.json create mode 120000 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server create mode 120000 plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud create mode 120000 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy create mode 120000 plugins/microsoft-sql-migration/skills/sqlpackage-import-export create mode 100644 plugins/microsoft-sql-ssms/.claude-plugin/plugin.json create mode 100644 plugins/microsoft-sql-ssms/.codex-plugin/plugin.json create mode 100644 plugins/microsoft-sql-ssms/.cursor-plugin/plugin.json create mode 100644 plugins/microsoft-sql-ssms/plugin.json create mode 120000 plugins/microsoft-sql-ssms/skills/capture-with-extended-events create mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks create mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure create mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-slow-query create mode 120000 plugins/microsoft-sql-ssms/skills/entra-id-auth create mode 120000 plugins/microsoft-sql-ssms/skills/read-execution-plan create mode 120000 plugins/microsoft-sql-ssms/skills/restore-and-recover create mode 120000 plugins/microsoft-sql-ssms/skills/rls-multi-tenant create mode 100644 plugins/microsoft-sql-vscode/.claude-plugin/plugin.json create mode 100644 plugins/microsoft-sql-vscode/.codex-plugin/plugin.json create mode 100644 plugins/microsoft-sql-vscode/.cursor-plugin/plugin.json create mode 100644 plugins/microsoft-sql-vscode/plugin.json create mode 120000 plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings create mode 120000 plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-dotnet create mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-python create mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node create mode 120000 plugins/microsoft-sql-vscode/skills/connect-to-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/design-azure-sql-schema create mode 120000 plugins/microsoft-sql-vscode/skills/diagnose-connection-errors create mode 120000 plugins/microsoft-sql-vscode/skills/ef-core-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/embeddings-and-external-models create mode 120000 plugins/microsoft-sql-vscode/skills/entra-id-auth create mode 120000 plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/prevent-sql-injection create mode 120000 plugins/microsoft-sql-vscode/skills/prisma-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/rag-on-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql create mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-correctness create mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson create mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge create mode 120000 plugins/microsoft-sql-vscode/skills/vector-search-azure-sql diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 481eef1..88d56f5 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -73,6 +73,34 @@ "./skills/t-sql-upserts-merge", "./skills/vector-search-azure-sql" ] + }, + { + "name": "microsoft-sql-vscode", + "displayName": "Microsoft SQL for Visual Studio Code", + "source": "./plugins/microsoft-sql-vscode", + "description": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "version": "0.1.0" + }, + { + "name": "microsoft-sql-ssms", + "displayName": "Microsoft SQL for SQL Server Management Studio", + "source": "./plugins/microsoft-sql-ssms", + "description": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "version": "0.1.0" + }, + { + "name": "microsoft-sql-migration", + "displayName": "Microsoft SQL migration", + "source": "./plugins/microsoft-sql-migration", + "description": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "version": "0.1.0" + }, + { + "name": "microsoft-sql-fdh", + "displayName": "Microsoft SQL provisioning and delivery", + "source": "./plugins/microsoft-sql-fdh", + "description": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "version": "0.1.0" } ] } diff --git a/plugins/microsoft-sql-fdh/.claude-plugin/plugin.json b/plugins/microsoft-sql-fdh/.claude-plugin/plugin.json new file mode 100644 index 0000000..5474b26 --- /dev/null +++ b/plugins/microsoft-sql-fdh/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL provisioning and delivery", + "name": "microsoft-sql-fdh", + "version": "0.1.0", + "description": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-fdh/.codex-plugin/plugin.json b/plugins/microsoft-sql-fdh/.codex-plugin/plugin.json new file mode 100644 index 0000000..9d68c65 --- /dev/null +++ b/plugins/microsoft-sql-fdh/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "name": "microsoft-sql-fdh", + "version": "0.1.0", + "description": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "./skills/", + "interface": { + "displayName": "Microsoft SQL provisioning and delivery", + "shortDescription": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "category": "Databases", + "logo": "assets/plugin-logo.svg", + "composerIcon": "assets/plugin-logo.svg" + } +} diff --git a/plugins/microsoft-sql-fdh/.cursor-plugin/plugin.json b/plugins/microsoft-sql-fdh/.cursor-plugin/plugin.json new file mode 100644 index 0000000..055d481 --- /dev/null +++ b/plugins/microsoft-sql-fdh/.cursor-plugin/plugin.json @@ -0,0 +1,25 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL provisioning and delivery", + "name": "microsoft-sql-fdh", + "version": "0.1.0", + "description": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "skills/", + "logo": "assets/plugin-logo.svg" +} diff --git a/plugins/microsoft-sql-fdh/plugin.json b/plugins/microsoft-sql-fdh/plugin.json new file mode 100644 index 0000000..12bb60e --- /dev/null +++ b/plugins/microsoft-sql-fdh/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "microsoft-sql-fdh", + "version": "0.1.0", + "description": "EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, database projects, safe migrations, GitHub Actions, and deploying the application beside it.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure new file mode 120000 index 0000000..fa1cf7d --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure @@ -0,0 +1 @@ +../../../skills/deploy-app-to-azure \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql new file mode 120000 index 0000000..c08c1e4 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql @@ -0,0 +1 @@ +../../../skills/github-actions-for-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db new file mode 120000 index 0000000..0a66007 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db @@ -0,0 +1 @@ +../../../skills/provision-azure-sql-db \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/provision-hyperscale b/plugins/microsoft-sql-fdh/skills/provision-hyperscale new file mode 120000 index 0000000..69975b9 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-hyperscale @@ -0,0 +1 @@ +../../../skills/provision-hyperscale \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely new file mode 120000 index 0000000..fb347b2 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely @@ -0,0 +1 @@ +../../../skills/schema-migrations-safely \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/sql-database-projects b/plugins/microsoft-sql-fdh/skills/sql-database-projects new file mode 120000 index 0000000..a816459 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/sql-database-projects @@ -0,0 +1 @@ +../../../skills/sql-database-projects \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/.claude-plugin/plugin.json b/plugins/microsoft-sql-migration/.claude-plugin/plugin.json new file mode 100644 index 0000000..ff6ca4a --- /dev/null +++ b/plugins/microsoft-sql-migration/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL migration", + "name": "microsoft-sql-migration", + "version": "0.1.0", + "description": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-migration/.codex-plugin/plugin.json b/plugins/microsoft-sql-migration/.codex-plugin/plugin.json new file mode 100644 index 0000000..ef5c814 --- /dev/null +++ b/plugins/microsoft-sql-migration/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "name": "microsoft-sql-migration", + "version": "0.1.0", + "description": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "./skills/", + "interface": { + "displayName": "Microsoft SQL migration", + "shortDescription": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "category": "Databases", + "logo": "assets/plugin-logo.svg", + "composerIcon": "assets/plugin-logo.svg" + } +} diff --git a/plugins/microsoft-sql-migration/.cursor-plugin/plugin.json b/plugins/microsoft-sql-migration/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e501c80 --- /dev/null +++ b/plugins/microsoft-sql-migration/.cursor-plugin/plugin.json @@ -0,0 +1,25 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL migration", + "name": "microsoft-sql-migration", + "version": "0.1.0", + "description": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "skills/", + "logo": "assets/plugin-logo.svg" +} diff --git a/plugins/microsoft-sql-migration/plugin.json b/plugins/microsoft-sql-migration/plugin.json new file mode 100644 index 0000000..0828004 --- /dev/null +++ b/plugins/microsoft-sql-migration/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "microsoft-sql-migration", + "version": "0.1.0", + "description": "EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that the service does not, and the four ways to get the data across.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server new file mode 120000 index 0000000..d1be456 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server @@ -0,0 +1 @@ +../../../skills/azuresql-db-from-sql-server \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud new file mode 120000 index 0000000..9621bbc --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud @@ -0,0 +1 @@ +../../../skills/azuresql-db-local-to-cloud \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy new file mode 120000 index 0000000..9e46830 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy @@ -0,0 +1 @@ +../../../skills/bulk-load-and-bulk-copy \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export new file mode 120000 index 0000000..538e9e7 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export @@ -0,0 +1 @@ +../../../skills/sqlpackage-import-export \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/.claude-plugin/plugin.json b/plugins/microsoft-sql-ssms/.claude-plugin/plugin.json new file mode 100644 index 0000000..e16354d --- /dev/null +++ b/plugins/microsoft-sql-ssms/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL for SQL Server Management Studio", + "name": "microsoft-sql-ssms", + "version": "0.1.0", + "description": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-ssms/.codex-plugin/plugin.json b/plugins/microsoft-sql-ssms/.codex-plugin/plugin.json new file mode 100644 index 0000000..40db4f3 --- /dev/null +++ b/plugins/microsoft-sql-ssms/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "name": "microsoft-sql-ssms", + "version": "0.1.0", + "description": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "./skills/", + "interface": { + "displayName": "Microsoft SQL for SQL Server Management Studio", + "shortDescription": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "category": "Databases", + "logo": "assets/plugin-logo.svg", + "composerIcon": "assets/plugin-logo.svg" + } +} diff --git a/plugins/microsoft-sql-ssms/.cursor-plugin/plugin.json b/plugins/microsoft-sql-ssms/.cursor-plugin/plugin.json new file mode 100644 index 0000000..05ff384 --- /dev/null +++ b/plugins/microsoft-sql-ssms/.cursor-plugin/plugin.json @@ -0,0 +1,25 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL for SQL Server Management Studio", + "name": "microsoft-sql-ssms", + "version": "0.1.0", + "description": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "skills/", + "logo": "assets/plugin-logo.svg" +} diff --git a/plugins/microsoft-sql-ssms/plugin.json b/plugins/microsoft-sql-ssms/plugin.json new file mode 100644 index 0000000..5665aad --- /dev/null +++ b/plugins/microsoft-sql-ssms/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "microsoft-sql-ssms", + "version": "0.1.0", + "description": "EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, resource pressure, execution plans, extended events, restore, and row-level security.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events new file mode 120000 index 0000000..9b42126 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events @@ -0,0 +1 @@ +../../../skills/capture-with-extended-events \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks new file mode 120000 index 0000000..c77ea88 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks @@ -0,0 +1 @@ +../../../skills/diagnose-blocking-and-deadlocks \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure new file mode 120000 index 0000000..e8c542b --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure @@ -0,0 +1 @@ +../../../skills/diagnose-resource-pressure \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query new file mode 120000 index 0000000..a95d4af --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query @@ -0,0 +1 @@ +../../../skills/diagnose-slow-query \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth b/plugins/microsoft-sql-ssms/skills/entra-id-auth new file mode 120000 index 0000000..e45999d --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth @@ -0,0 +1 @@ +../../../skills/entra-id-auth \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/read-execution-plan b/plugins/microsoft-sql-ssms/skills/read-execution-plan new file mode 120000 index 0000000..f1ab778 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/read-execution-plan @@ -0,0 +1 @@ +../../../skills/read-execution-plan \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/restore-and-recover b/plugins/microsoft-sql-ssms/skills/restore-and-recover new file mode 120000 index 0000000..4ef369b --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/restore-and-recover @@ -0,0 +1 @@ +../../../skills/restore-and-recover \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant new file mode 120000 index 0000000..f1b3cb7 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant @@ -0,0 +1 @@ +../../../skills/rls-multi-tenant \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/.claude-plugin/plugin.json b/plugins/microsoft-sql-vscode/.claude-plugin/plugin.json new file mode 100644 index 0000000..6dbd2d5 --- /dev/null +++ b/plugins/microsoft-sql-vscode/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL for Visual Studio Code", + "name": "microsoft-sql-vscode", + "version": "0.1.0", + "description": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-vscode/.codex-plugin/plugin.json b/plugins/microsoft-sql-vscode/.codex-plugin/plugin.json new file mode 100644 index 0000000..fcf9707 --- /dev/null +++ b/plugins/microsoft-sql-vscode/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "name": "microsoft-sql-vscode", + "version": "0.1.0", + "description": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "./skills/", + "interface": { + "displayName": "Microsoft SQL for Visual Studio Code", + "shortDescription": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "category": "Databases", + "logo": "assets/plugin-logo.svg", + "composerIcon": "assets/plugin-logo.svg" + } +} diff --git a/plugins/microsoft-sql-vscode/.cursor-plugin/plugin.json b/plugins/microsoft-sql-vscode/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e7bf3fe --- /dev/null +++ b/plugins/microsoft-sql-vscode/.cursor-plugin/plugin.json @@ -0,0 +1,25 @@ +{ + "$comment": "GENERATED by scripts/generate.mjs from catalog/catalog.json. Do not edit by hand; the parity gate will fail.", + "displayName": "Microsoft SQL for Visual Studio Code", + "name": "microsoft-sql-vscode", + "version": "0.1.0", + "description": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ], + "skills": "skills/", + "logo": "assets/plugin-logo.svg" +} diff --git a/plugins/microsoft-sql-vscode/plugin.json b/plugins/microsoft-sql-vscode/plugin.json new file mode 100644 index 0000000..205804f --- /dev/null +++ b/plugins/microsoft-sql-vscode/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "microsoft-sql-vscode", + "version": "0.1.0", + "description": "EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder skills, which clash with the Data API builder tooling GitHub Copilot already carries there.", + "author": { + "name": "Microsoft", + "url": "https://microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-sql-skills", + "repository": "https://github.com/microsoft/azure-sql-skills", + "license": "MIT", + "keywords": [ + "azure-sql", + "sql", + "database", + "tsql", + "agent-skills", + "vector-search", + "entra-id" + ] +} diff --git a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings new file mode 120000 index 0000000..ef276ee --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings @@ -0,0 +1 @@ +../../../skills/azure-functions-sql-bindings \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql new file mode 120000 index 0000000..788abb5 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql @@ -0,0 +1 @@ +../../../skills/build-app-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet new file mode 120000 index 0000000..80fd47c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet @@ -0,0 +1 @@ +../../../skills/connect-from-dotnet \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-python b/plugins/microsoft-sql-vscode/skills/connect-from-python new file mode 120000 index 0000000..95a1bba --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-python @@ -0,0 +1 @@ +../../../skills/connect-from-python \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node b/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node new file mode 120000 index 0000000..6de9103 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node @@ -0,0 +1 @@ +../../../skills/connect-from-typescript-and-node \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql new file mode 120000 index 0000000..f271ea1 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql @@ -0,0 +1 @@ +../../../skills/connect-to-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema new file mode 120000 index 0000000..61a034f --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema @@ -0,0 +1 @@ +../../../skills/design-azure-sql-schema \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors new file mode 120000 index 0000000..579ec97 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors @@ -0,0 +1 @@ +../../../skills/diagnose-connection-errors \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql new file mode 120000 index 0000000..a32141d --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql @@ -0,0 +1 @@ +../../../skills/ef-core-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models new file mode 120000 index 0000000..cb2ca2e --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models @@ -0,0 +1 @@ +../../../skills/embeddings-and-external-models \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth b/plugins/microsoft-sql-vscode/skills/entra-id-auth new file mode 120000 index 0000000..e45999d --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth @@ -0,0 +1 @@ +../../../skills/entra-id-auth \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql new file mode 120000 index 0000000..1660d3c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql @@ -0,0 +1 @@ +../../../skills/langchain-and-llamaindex-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection new file mode 120000 index 0000000..283fe79 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection @@ -0,0 +1 @@ +../../../skills/prevent-sql-injection \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/prisma-azure-sql b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql new file mode 120000 index 0000000..d90e52e --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql @@ -0,0 +1 @@ +../../../skills/prisma-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql new file mode 120000 index 0000000..56fc4f8 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql @@ -0,0 +1 @@ +../../../skills/rag-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql new file mode 120000 index 0000000..bd14ba7 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql @@ -0,0 +1 @@ +../../../skills/sqlalchemy-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-correctness b/plugins/microsoft-sql-vscode/skills/t-sql-correctness new file mode 120000 index 0000000..883f308 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-correctness @@ -0,0 +1 @@ +../../../skills/t-sql-correctness \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson new file mode 120000 index 0000000..281b8aa --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson @@ -0,0 +1 @@ +../../../skills/t-sql-json-and-openjson \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge new file mode 120000 index 0000000..691abda --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge @@ -0,0 +1 @@ +../../../skills/t-sql-upserts-merge \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql new file mode 120000 index 0000000..e3c7b65 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql @@ -0,0 +1 @@ +../../../skills/vector-search-azure-sql \ No newline at end of file diff --git a/scripts/generate.mjs b/scripts/generate.mjs index 5582fcf..a533ff3 100644 --- a/scripts/generate.mjs +++ b/scripts/generate.mjs @@ -11,7 +11,7 @@ // manifests hide in dot-directories as .json. Generating them from one source // makes drift a build failure instead of a discovery. -import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'node:fs'; +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, symlinkSync, readlinkSync, rmSync, cpSync } from 'node:fs'; import { join, dirname } from 'node:path'; const CATALOG = JSON.parse(readFileSync('catalog/catalog.json', 'utf8')); @@ -250,12 +250,197 @@ const j = (o) => JSON.stringify({ $comment: BANNER, ...o }, null, 2) + '\n'; const STORE_NAME = 'Azure SQL'; emit('.claude-plugin/plugin.json', j({ displayName: STORE_NAME, ...base })); + +// --------------------------------------------------------------------------- +// EXPERIMENTAL, 2026-09-21: tool-specific plugins carrying subsets of the one +// skills tree. +// +// The earlier experiment on experiment-scoped-collections put every entry at +// `source: "./"` and gave each its own `skills` list. Claude Code honoured it. +// GitHub Copilot CLI printed a filtered count and then loaded all 57 under +// every entry, Cursor ignored the list, and Codex refused the install outright +// with "plugin.json name 'azure-sql' does not match marketplace plugin name +// 'azure-sql-vscode'". +// +// This is the approach that error asked for: a FOLDER PER PLUGIN under +// plugins/, each with its own plugin.json whose `name` matches its marketplace +// entry, and each with its own skills/ holding only that plugin's skills. +// +// The open question is how plugins//skills/ gets content without a second +// hand-maintained copy. Two sources settle what is allowed: +// +// Agent Plugins specification, agent-plugins.org/specification: +// "The fixed discovery location is skills/." +// "plugin.json cannot override these locations or contain inline component +// configuration." +// "Its schema is closed: the only permitted top-level fields are $schema, +// name, version, description, author, homepage, repository, license, +// keywords, and extensions." +// So a `skills` field pointing back at ../../skills is not merely rejected, +// it is not a field. That closes approach 2 by design. +// +// The same specification on symlinks: +// "Symlinks, junctions, reparse points, and equivalent filesystem +// mechanisms MAY resolve to targets within the plugin root, but clients +// MUST reject package paths that resolve outside it." +// A link from plugins//skills/ to ../../../skills/ +// escapes the PLUGIN root while staying inside the MARKETPLACE root. +// +// Claude Code, code.claude.com/docs/en/plugin-marketplaces, on caching a +// marketplace that does exactly that: +// "Elsewhere within the same marketplace: the symlink is dereferenced. The +// target's content is copied into the cache in its place" +// So Claude Code resolves the link away before discovery ever runs. Whether +// any other client does is measured, not assumed. +// +// SUBSET_SHARING selects what is written into plugins//skills/. +// 'symlink' one relative symlink per skill, pointing at the canonical tree +// 'copy' a byte-for-byte copy, regenerated here and diffed by --check +// Nothing else changes between the two, so the clients can be measured against +// the same manifests. +// --------------------------------------------------------------------------- +const SUBSET_SHARING = 'symlink'; + +// Data API builder is excluded from the Visual Studio Code plugin because it +// clashes with Data API builder tooling GitHub Copilot already carries there. +const DAB_SKILLS = ['dab-rest-and-graphql', 'azuresql-db-dab']; + +// The membership is a first defensible cut, not a decided one. Each list is a +// job someone does in that tool, and each is pinned to an exact size so a +// silent drop or a rename fails the build instead of shipping a short plugin. +const SUBSETS = [ + { + name: 'microsoft-sql-vscode', + displayName: 'Microsoft SQL for Visual Studio Code', + description: + 'EXPERIMENTAL. Building an application against Azure SQL Database inside Visual Studio Code: ' + + 'drivers, ORMs, T-SQL correctness, vector search and retrieval. Without the Data API builder ' + + 'skills, which clash with the Data API builder tooling GitHub Copilot already carries there.', + skills: [ + 'connect-to-azure-sql', 'connect-from-dotnet', 'connect-from-python', + 'connect-from-typescript-and-node', 'diagnose-connection-errors', + 'design-azure-sql-schema', 'ef-core-azure-sql', 'prisma-azure-sql', 'sqlalchemy-azure-sql', + 't-sql-correctness', 't-sql-json-and-openjson', 't-sql-upserts-merge', + 'prevent-sql-injection', 'entra-id-auth', + 'vector-search-azure-sql', 'embeddings-and-external-models', 'rag-on-azure-sql', + 'langchain-and-llamaindex-on-azure-sql', + 'build-app-on-azure-sql', 'azure-functions-sql-bindings', + ], + }, + { + name: 'microsoft-sql-ssms', + displayName: 'Microsoft SQL for SQL Server Management Studio', + description: + 'EXPERIMENTAL. Operating and diagnosing a running Azure SQL Database: slow queries, blocking, ' + + 'resource pressure, execution plans, extended events, restore, and row-level security.', + skills: [ + 'diagnose-slow-query', 'diagnose-blocking-and-deadlocks', 'diagnose-resource-pressure', + 'read-execution-plan', 'capture-with-extended-events', 'restore-and-recover', + 'rls-multi-tenant', 'entra-id-auth', + ], + }, + { + name: 'microsoft-sql-migration', + displayName: 'Microsoft SQL migration', + description: + 'EXPERIMENTAL. Moving an existing database to Azure SQL Database: what SQL Server does that ' + + 'the service does not, and the four ways to get the data across.', + skills: [ + 'azuresql-db-from-sql-server', 'azuresql-db-local-to-cloud', + 'sqlpackage-import-export', 'bulk-load-and-bulk-copy', + ], + }, + { + name: 'microsoft-sql-fdh', + displayName: 'Microsoft SQL provisioning and delivery', + description: + 'EXPERIMENTAL. Standing a database up and keeping schema moving: provisioning, Hyperscale, ' + + 'database projects, safe migrations, GitHub Actions, and deploying the application beside it.', + skills: [ + 'provision-azure-sql-db', 'provision-hyperscale', 'deploy-app-to-azure', + 'sql-database-projects', 'schema-migrations-safely', 'github-actions-for-sql', + ], + }, +]; + +// Fail loudly rather than emit a plugin that resolves to nothing, to something +// short, or to the whole catalog. An install that resolves the wrong set is the +// failure mode this repository keeps finding late. +const EXPECTED = { 'microsoft-sql-vscode': 20, 'microsoft-sql-ssms': 8, 'microsoft-sql-migration': 4, 'microsoft-sql-fdh': 6 }; +const onDisk = new Set(present.map((p) => p.name)); +for (const s of SUBSETS) { + const missing = s.skills.filter((id) => !onDisk.has(id)); + if (missing.length) { + console.error(`x ${s.name} lists ${missing.length} skill(s) not on disk: ${missing.join(', ')}`); + process.exit(1); + } + if (new Set(s.skills).size !== s.skills.length) { + console.error(`x ${s.name} lists a skill twice`); + process.exit(1); + } + if (s.skills.length !== EXPECTED[s.name]) { + console.error(`x ${s.name} resolved ${s.skills.length} skills, expected ${EXPECTED[s.name]}`); + process.exit(1); + } +} +const vscode = SUBSETS.find((s) => s.name === 'microsoft-sql-vscode'); +for (const dab of DAB_SKILLS) { + if (vscode.skills.includes(dab)) { + console.error(`x ${dab} must not appear in ${vscode.name}`); + process.exit(1); + } +} + +// Each subset folder gets the portable plugin.json, whose `name` matches the +// marketplace entry, plus the per-tool manifests so a storefront has a display +// name here too. No `skills` key anywhere: the specification fixes discovery at +// skills/ and forbids a manifest from saying otherwise. +for (const s of SUBSETS) { + const dir = `plugins/${s.name}`; + const sub = { ...base, name: s.name, description: s.description }; + emit(`${dir}/plugin.json`, JSON.stringify({ + $schema: 'https://agent-plugins.org/schemas/1.0.0/plugin.schema.json', + name: s.name, + version: PKG.version, + description: s.description, + author: { name: 'Microsoft', url: 'https://microsoft.com' }, + homepage: HOMEPAGE, + repository: REPO, + license: 'MIT', + keywords: KEYWORDS, + }, null, 2) + '\n'); + emit(`${dir}/.claude-plugin/plugin.json`, j({ displayName: s.displayName, ...sub })); + emit(`${dir}/.codex-plugin/plugin.json`, j({ + ...sub, + skills: './skills/', + interface: { + displayName: s.displayName, + shortDescription: s.description, + category: 'Databases', + logo: 'assets/plugin-logo.svg', + composerIcon: 'assets/plugin-logo.svg', + }, + })); + emit(`${dir}/.cursor-plugin/plugin.json`, j({ displayName: s.displayName, ...sub, skills: 'skills/', logo: 'assets/plugin-logo.svg' })); +} + emit('.claude-plugin/marketplace.json', j({ name: NAME, owner: { name: 'Microsoft', url: REPO }, metadata: { description: SUMMARY }, - // Wave 2 adds one entry per persona, each with its own skills array. - plugins: [{ name: NAME, source: './', description: SUMMARY, version: PKG.version, skills: skillPaths }], + plugins: [ + { name: NAME, source: './', description: SUMMARY, version: PKG.version, skills: skillPaths }, + // No `skills` array on these. Their source is a real plugin folder, so the + // default skills/ scan inside it is the whole declaration, which is the one + // thing every client in the measurement agrees on. + ...SUBSETS.map((s) => ({ + name: s.name, + displayName: s.displayName, + source: `./plugins/${s.name}`, + description: s.description, + version: PKG.version, + })), + ], })); // PER-TOOL MANIFESTS, and why there are now two of them. @@ -391,7 +576,83 @@ emit('plugin.json', JSON.stringify({ // finding. It lands when the endpoint does. // --------------------------------------------------------------------------- -let stale = 0; +// --------------------------------------------------------------------------- +// plugins//skills/, materialised. EXPERIMENTAL, see SUBSET_SHARING. +// +// This is the part `emit` cannot do, because it writes file bodies and what is +// needed here is either a symlink or a whole subtree. Both modes are checkable: +// --check reports a difference rather than repairing it, the same contract the +// rest of this script keeps. +// --------------------------------------------------------------------------- +let subsetStale = 0; +const subsetNote = (msg) => { + subsetStale++; + if (check) console.error(`x ${msg}`); +}; + +const listDir = (p) => (existsSync(p) ? readdirSync(p).filter((n) => n !== '.DS_Store').sort() : []); + +// A skill directory, flattened to [relative path, contents] pairs, so two trees +// can be compared without walking them in lockstep. +const flatten = (root, prefix = '') => { + const out = []; + for (const name of readdirSync(root).filter((n) => n !== '.DS_Store').sort()) { + const full = join(root, name); + if (statSync(full).isDirectory()) out.push(...flatten(full, `${prefix}${name}/`)); + else out.push([`${prefix}${name}`, readFileSync(full)]); + } + return out; +}; + +for (const s of SUBSETS) { + const dir = join('plugins', s.name, 'skills'); + const want = [...s.skills].sort(); + + if (!check) { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + } + + const have = listDir(dir); + if (have.join(',') !== want.join(',')) subsetNote(`${dir} holds [${have.join(', ')}], expected [${want.join(', ')}]`); + + for (const id of want) { + const dest = join(dir, id); + const source = join('skills', id); + + if (SUBSET_SHARING === 'symlink') { + // Relative, and up three levels: plugins//skills/ back to + // skills/. Relative rather than absolute so a clone on another + // machine resolves, and so git stores the link rather than a path from + // this laptop. + const target = `../../../skills/${id}`; + if (check) { + let actual = null; + try { actual = readlinkSync(dest); } catch { /* not a symlink, or absent */ } + if (actual !== target) subsetNote(`${dest} should be a symlink to ${target}, found ${actual ?? 'a non-symlink or nothing'}`); + } else { + symlinkSync(target, dest); + } + } else if (SUBSET_SHARING === 'copy') { + if (check) { + if (!existsSync(dest)) { subsetNote(`${dest} is missing`); continue; } + const a = flatten(source); + const b = flatten(dest); + if (a.length !== b.length || a.some(([p, body], i) => b[i][0] !== p || !body.equals(b[i][1]))) { + subsetNote(`${dest} differs from skills/${id}`); + } + } else { + cpSync(source, dest, { recursive: true }); + } + } else { + console.error(`x SUBSET_SHARING is "${SUBSET_SHARING}", which is not a mode`); + process.exit(1); + } + } + if (!check) console.log(` wrote ${dir} (${want.length} skills, ${SUBSET_SHARING})`); +} + +let stale = subsetStale; for (const [path, body] of outputs) { const current = existsSync(path) ? readFileSync(path, 'utf8') : null; if (current === body) continue; From 4cfcffabb107cffb82effef705b3d30da46d9af0 Mon Sep 17 00:00:00 2001 From: Carlos Robles Date: Mon, 21 Sep 2026 22:22:37 -0600 Subject: [PATCH 2/3] Fall back to generated copies, since Codex drops the symlinks --- .../skills/deploy-app-to-azure | 1 - .../skills/deploy-app-to-azure/SKILL.md | 394 +++++++++++++++++ .../references/azd-and-azure-sql.md | 207 +++++++++ .../deploy-app-to-azure/skill.spec.jsonc | 204 +++++++++ .../skills/github-actions-for-sql | 1 - .../skills/github-actions-for-sql/SKILL.md | 310 +++++++++++++ .../references/oidc-and-firewall.md | 230 ++++++++++ .../github-actions-for-sql/skill.spec.jsonc | 152 +++++++ .../skills/provision-azure-sql-db | 1 - .../skills/provision-azure-sql-db/SKILL.md | 198 +++++++++ .../provision-azure-sql-db/skill.spec.jsonc | 115 +++++ .../skills/provision-hyperscale | 1 - .../skills/provision-hyperscale/SKILL.md | 327 ++++++++++++++ .../references/exit-paths-and-eligibility.md | 137 ++++++ .../provision-hyperscale/skill.spec.jsonc | 130 ++++++ .../skills/schema-migrations-safely | 1 - .../skills/schema-migrations-safely/SKILL.md | 348 +++++++++++++++ .../alter-cost-schema-locks-and-retry.md | 360 +++++++++++++++ .../schema-migrations-safely/skill.spec.jsonc | 348 +++++++++++++++ .../skills/sql-database-projects | 1 - .../skills/sql-database-projects/SKILL.md | 306 +++++++++++++ .../target-platform-and-refactorlog-runs.md | 185 ++++++++ .../sql-database-projects/skill.spec.jsonc | 158 +++++++ .../skills/azuresql-db-from-sql-server | 1 - .../azuresql-db-from-sql-server/SKILL.md | 220 ++++++++++ .../references/migrate-compose.md | 98 +++++ .../sql-server-vs-azure-feature-matrix.md | 69 +++ .../skill.spec.jsonc | 139 ++++++ .../skills/azuresql-db-local-to-cloud | 1 - .../azuresql-db-local-to-cloud/SKILL.md | 296 +++++++++++++ .../references/auth-local-vs-cloud.md | 198 +++++++++ .../skill.spec.jsonc | 115 +++++ .../skills/bulk-load-and-bulk-copy | 1 - .../skills/bulk-load-and-bulk-copy/SKILL.md | 242 ++++++++++ .../bulk-load-errors-and-log-rate-governor.md | 228 ++++++++++ .../references/driver-bulk-apis.md | 76 ++++ .../bulk-load-and-bulk-copy/skill.spec.jsonc | 142 ++++++ .../skills/sqlpackage-import-export | 1 - .../skills/sqlpackage-import-export/SKILL.md | 249 +++++++++++ .../dacpac-bacpac-contents-and-errors.md | 181 ++++++++ .../references/preflight-and-sql71627.md | 76 ++++ .../sqlpackage-import-export/skill.spec.jsonc | 152 +++++++ .../skills/capture-with-extended-events | 1 - .../capture-with-extended-events/SKILL.md | 311 +++++++++++++ .../references/event-file-to-blob-storage.md | 111 +++++ .../sessions-that-capture-nothing.md | 174 ++++++++ .../skill.spec.jsonc | 166 +++++++ .../skills/diagnose-blocking-and-deadlocks | 1 - .../diagnose-blocking-and-deadlocks/SKILL.md | 210 +++++++++ ...blocking-visibility-and-deadlock-graphs.md | 237 ++++++++++ .../skill.spec.jsonc | 193 ++++++++ .../skills/diagnose-resource-pressure | 1 - .../diagnose-resource-pressure/SKILL.md | 192 ++++++++ .../references/dmv-and-limits-reference.md | 158 +++++++ .../skill.spec.jsonc | 171 ++++++++ .../skills/diagnose-slow-query | 1 - .../skills/diagnose-slow-query/SKILL.md | 270 ++++++++++++ .../references/query-store-queries.md | 131 ++++++ .../diagnose-slow-query/skill.spec.jsonc | 134 ++++++ .../microsoft-sql-ssms/skills/entra-id-auth | 1 - .../skills/entra-id-auth/SKILL.md | 382 ++++++++++++++++ .../references/driver-auth-keywords.md | 214 +++++++++ .../references/entra-only-and-policy.md | 140 ++++++ .../references/identity-errors.md | 230 ++++++++++ .../skills/entra-id-auth/skill.spec.jsonc | 148 +++++++ .../skills/read-execution-plan | 1 - .../skills/read-execution-plan/SKILL.md | 216 +++++++++ .../references/dmv-and-xquery-reference.md | 204 +++++++++ .../read-execution-plan/skill.spec.jsonc | 179 ++++++++ .../skills/restore-and-recover | 1 - .../skills/restore-and-recover/SKILL.md | 240 ++++++++++ .../references/backup-restore-tsql-errors.md | 75 ++++ .../restore-and-recover/skill.spec.jsonc | 138 ++++++ .../skills/rls-multi-tenant | 1 - .../skills/rls-multi-tenant/SKILL.md | 391 +++++++++++++++++ .../block-predicates-pooling-and-bypass.md | 412 ++++++++++++++++++ .../skills/rls-multi-tenant/skill.spec.jsonc | 186 ++++++++ .../skills/azure-functions-sql-bindings | 1 - .../azure-functions-sql-bindings/SKILL.md | 273 ++++++++++++ .../references/bindings-by-language.md | 183 ++++++++ .../skill.spec.jsonc | 201 +++++++++ .../skills/build-app-on-azure-sql | 1 - .../skills/build-app-on-azure-sql/SKILL.md | 335 ++++++++++++++ .../build-app-on-azure-sql/skill.spec.jsonc | 149 +++++++ .../skills/connect-from-dotnet | 1 - .../skills/connect-from-dotnet/SKILL.md | 225 ++++++++++ .../entra-modes-and-connection-keywords.md | 57 +++ .../connect-from-dotnet/skill.spec.jsonc | 126 ++++++ .../skills/connect-from-python | 1 - .../skills/connect-from-python/SKILL.md | 265 +++++++++++ .../odbc-driver-install-on-linux.md | 57 +++ .../connect-from-python/skill.spec.jsonc | 132 ++++++ .../skills/connect-from-typescript-and-node | 1 - .../connect-from-typescript-and-node/SKILL.md | 207 +++++++++ .../skill.spec.jsonc | 153 +++++++ .../skills/connect-to-azure-sql | 1 - .../skills/connect-to-azure-sql/SKILL.md | 337 ++++++++++++++ .../connect-to-azure-sql/skill.spec.jsonc | 144 ++++++ .../skills/design-azure-sql-schema | 1 - .../skills/design-azure-sql-schema/SKILL.md | 346 +++++++++++++++ .../key-bytes-collation-and-identity-gaps.md | 395 +++++++++++++++++ .../design-azure-sql-schema/skill.spec.jsonc | 345 +++++++++++++++ .../skills/diagnose-connection-errors | 1 - .../diagnose-connection-errors/SKILL.md | 218 +++++++++ .../references/error-codes.md | 265 +++++++++++ .../skill.spec.jsonc | 141 ++++++ .../skills/ef-core-azure-sql | 1 - .../skills/ef-core-azure-sql/SKILL.md | 355 +++++++++++++++ .../retry-transactions-and-type-mapping.md | 378 ++++++++++++++++ .../skills/ef-core-azure-sql/skill.spec.jsonc | 143 ++++++ .../skills/embeddings-and-external-models | 1 - .../embeddings-and-external-models/SKILL.md | 333 ++++++++++++++ .../references/measured-external-model.md | 258 +++++++++++ .../skill.spec.jsonc | 273 ++++++++++++ .../microsoft-sql-vscode/skills/entra-id-auth | 1 - .../skills/entra-id-auth/SKILL.md | 382 ++++++++++++++++ .../references/driver-auth-keywords.md | 214 +++++++++ .../references/entra-only-and-policy.md | 140 ++++++ .../references/identity-errors.md | 230 ++++++++++ .../skills/entra-id-auth/skill.spec.jsonc | 148 +++++++ .../langchain-and-llamaindex-on-azure-sql | 1 - .../SKILL.md | 356 +++++++++++++++ .../framework-prompts-and-vector-store.md | 297 +++++++++++++ .../skill.spec.jsonc | 260 +++++++++++ .../skills/prevent-sql-injection | 1 - .../skills/prevent-sql-injection/SKILL.md | 211 +++++++++ .../quotename-null-and-order-by-clashes.md | 211 +++++++++ .../prevent-sql-injection/skill.spec.jsonc | 196 +++++++++ .../skills/prisma-azure-sql | 1 - .../skills/prisma-azure-sql/SKILL.md | 293 +++++++++++++ .../skills/prisma-azure-sql/skill.spec.jsonc | 131 ++++++ .../skills/rag-on-azure-sql | 1 - .../skills/rag-on-azure-sql/SKILL.md | 311 +++++++++++++ .../rag-on-azure-sql/references/pipeline.md | 175 ++++++++ .../skills/rag-on-azure-sql/skill.spec.jsonc | 211 +++++++++ .../skills/sqlalchemy-azure-sql | 1 - .../skills/sqlalchemy-azure-sql/SKILL.md | 273 ++++++++++++ .../sqlalchemy-azure-sql/skill.spec.jsonc | 155 +++++++ .../skills/t-sql-correctness | 1 - .../skills/t-sql-correctness/SKILL.md | 224 ++++++++++ .../references/postgres-to-tsql.md | 151 +++++++ .../skills/t-sql-correctness/skill.spec.jsonc | 253 +++++++++++ .../skills/t-sql-json-and-openjson | 1 - .../skills/t-sql-json-and-openjson/SKILL.md | 190 ++++++++ .../json-type-errors-and-timings.md | 304 +++++++++++++ .../t-sql-json-and-openjson/skill.spec.jsonc | 336 ++++++++++++++ .../skills/t-sql-upserts-merge | 1 - .../skills/t-sql-upserts-merge/SKILL.md | 231 ++++++++++ .../merge-hazards-and-concurrency.md | 267 ++++++++++++ .../t-sql-upserts-merge/skill.spec.jsonc | 186 ++++++++ .../skills/vector-search-azure-sql | 1 - .../skills/vector-search-azure-sql/SKILL.md | 348 +++++++++++++++ .../vector-restrictions-and-query-plans.md | 272 ++++++++++++ .../vector-search-azure-sql/skill.spec.jsonc | 404 +++++++++++++++++ scripts/generate.mjs | 2 +- 155 files changed, 25858 insertions(+), 39 deletions(-) delete mode 120000 plugins/microsoft-sql-fdh/skills/deploy-app-to-azure create mode 100644 plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/references/azd-and-azure-sql.md create mode 100644 plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-fdh/skills/github-actions-for-sql create mode 100644 plugins/microsoft-sql-fdh/skills/github-actions-for-sql/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/github-actions-for-sql/references/oidc-and-firewall.md create mode 100644 plugins/microsoft-sql-fdh/skills/github-actions-for-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-fdh/skills/provision-azure-sql-db create mode 100644 plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-fdh/skills/provision-hyperscale create mode 100644 plugins/microsoft-sql-fdh/skills/provision-hyperscale/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/provision-hyperscale/references/exit-paths-and-eligibility.md create mode 100644 plugins/microsoft-sql-fdh/skills/provision-hyperscale/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-fdh/skills/schema-migrations-safely create mode 100644 plugins/microsoft-sql-fdh/skills/schema-migrations-safely/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/schema-migrations-safely/references/alter-cost-schema-locks-and-retry.md create mode 100644 plugins/microsoft-sql-fdh/skills/schema-migrations-safely/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-fdh/skills/sql-database-projects create mode 100644 plugins/microsoft-sql-fdh/skills/sql-database-projects/SKILL.md create mode 100644 plugins/microsoft-sql-fdh/skills/sql-database-projects/references/target-platform-and-refactorlog-runs.md create mode 100644 plugins/microsoft-sql-fdh/skills/sql-database-projects/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server create mode 100755 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/SKILL.md create mode 100755 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/migrate-compose.md create mode 100755 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/sql-server-vs-azure-feature-matrix.md create mode 100644 plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud create mode 100755 plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/SKILL.md create mode 100755 plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/references/auth-local-vs-cloud.md create mode 100644 plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy create mode 100644 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/SKILL.md create mode 100644 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/bulk-load-errors-and-log-rate-governor.md create mode 100644 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/driver-bulk-apis.md create mode 100644 plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-migration/skills/sqlpackage-import-export create mode 100644 plugins/microsoft-sql-migration/skills/sqlpackage-import-export/SKILL.md create mode 100644 plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/dacpac-bacpac-contents-and-errors.md create mode 100644 plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/preflight-and-sql71627.md create mode 100644 plugins/microsoft-sql-migration/skills/sqlpackage-import-export/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/capture-with-extended-events create mode 100644 plugins/microsoft-sql-ssms/skills/capture-with-extended-events/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/event-file-to-blob-storage.md create mode 100644 plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/sessions-that-capture-nothing.md create mode 100644 plugins/microsoft-sql-ssms/skills/capture-with-extended-events/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/references/blocking-visibility-and-deadlock-graphs.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/references/dmv-and-limits-reference.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/diagnose-slow-query create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-slow-query/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-slow-query/references/query-store-queries.md create mode 100644 plugins/microsoft-sql-ssms/skills/diagnose-slow-query/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/entra-id-auth create mode 100644 plugins/microsoft-sql-ssms/skills/entra-id-auth/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/entra-id-auth/references/driver-auth-keywords.md create mode 100644 plugins/microsoft-sql-ssms/skills/entra-id-auth/references/entra-only-and-policy.md create mode 100644 plugins/microsoft-sql-ssms/skills/entra-id-auth/references/identity-errors.md create mode 100644 plugins/microsoft-sql-ssms/skills/entra-id-auth/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/read-execution-plan create mode 100644 plugins/microsoft-sql-ssms/skills/read-execution-plan/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/read-execution-plan/references/dmv-and-xquery-reference.md create mode 100644 plugins/microsoft-sql-ssms/skills/read-execution-plan/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/restore-and-recover create mode 100644 plugins/microsoft-sql-ssms/skills/restore-and-recover/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/restore-and-recover/references/backup-restore-tsql-errors.md create mode 100644 plugins/microsoft-sql-ssms/skills/restore-and-recover/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-ssms/skills/rls-multi-tenant create mode 100644 plugins/microsoft-sql-ssms/skills/rls-multi-tenant/SKILL.md create mode 100644 plugins/microsoft-sql-ssms/skills/rls-multi-tenant/references/block-predicates-pooling-and-bypass.md create mode 100644 plugins/microsoft-sql-ssms/skills/rls-multi-tenant/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings create mode 100644 plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/references/bindings-by-language.md create mode 100644 plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-dotnet create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-dotnet/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-dotnet/references/entra-modes-and-connection-keywords.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-dotnet/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-python create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-python/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-python/references/odbc-driver-install-on-linux.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-python/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/connect-to-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/design-azure-sql-schema create mode 100644 plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/references/key-bytes-collation-and-identity-gaps.md create mode 100644 plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/diagnose-connection-errors create mode 100644 plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/references/error-codes.md create mode 100644 plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/ef-core-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/references/retry-transactions-and-type-mapping.md create mode 100644 plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/embeddings-and-external-models create mode 100644 plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/references/measured-external-model.md create mode 100644 plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/entra-id-auth create mode 100644 plugins/microsoft-sql-vscode/skills/entra-id-auth/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/entra-id-auth/references/driver-auth-keywords.md create mode 100644 plugins/microsoft-sql-vscode/skills/entra-id-auth/references/entra-only-and-policy.md create mode 100644 plugins/microsoft-sql-vscode/skills/entra-id-auth/references/identity-errors.md create mode 100644 plugins/microsoft-sql-vscode/skills/entra-id-auth/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/references/framework-prompts-and-vector-store.md create mode 100644 plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/prevent-sql-injection create mode 100644 plugins/microsoft-sql-vscode/skills/prevent-sql-injection/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/prevent-sql-injection/references/quotename-null-and-order-by-clashes.md create mode 100644 plugins/microsoft-sql-vscode/skills/prevent-sql-injection/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/prisma-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/prisma-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/prisma-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/rag-on-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/references/pipeline.md create mode 100644 plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-correctness create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-correctness/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-correctness/references/postgres-to-tsql.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-correctness/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/references/json-type-errors-and-timings.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/references/merge-hazards-and-concurrency.md create mode 100644 plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/skill.spec.jsonc delete mode 120000 plugins/microsoft-sql-vscode/skills/vector-search-azure-sql create mode 100644 plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/SKILL.md create mode 100644 plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/references/vector-restrictions-and-query-plans.md create mode 100644 plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/skill.spec.jsonc diff --git a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure deleted file mode 120000 index fa1cf7d..0000000 --- a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure +++ /dev/null @@ -1 +0,0 @@ -../../../skills/deploy-app-to-azure \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/SKILL.md b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/SKILL.md new file mode 100644 index 0000000..5ce7a9e --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/SKILL.md @@ -0,0 +1,394 @@ +--- +name: deploy-app-to-azure +description: >- + Takes a working local application and its Azure SQL Database to Azure with the Azure Developer + CLI, reading its infrastructure rather than inheriting it. The one first-party template pairing + a web app with Azure SQL Database uses a database password and grants db_owner, and the + firewall rule it ships under an Azure services name spans the whole public address space. Use + when a user says "deploy my app to Azure", "azd up", "which azd template should I start from", + "get this into Azure without a password", or when a deployment reported success and the app + then fails its first database call with a login failure. Covers what each template does about + identity, what init, provision, deploy and up each do, and what teardown leaves behind. + github-actions-for-sql owns pipelines, provision-azure-sql-db creating the server and database, + entra-id-auth the database user and the grant. +--- + +# Deploy an application and its Azure SQL Database to Azure + +**This owns the deployment sequence, and the infrastructure handed to you at the start of it.** It +does not create the database, it does not own the identity, and it is not the pipeline. + +Verified 2026-09-03 against the Azure Developer CLI 1.32.0, Azure CLI 2.90.0, sqlcmd 1.10.0 and +.NET 8.0.421, by listing the template gallery and reading the shipped Bicep of every +Microsoft-published template it returns for Azure SQL Database. + +## The correction: a first-party template's infrastructure is not a reviewed baseline + +An agent treats a Microsoft sample's `infra` directory as settled, spends its attention on the +application code, and asked to reach Azure SQL Database without a password recommends the +first-party template as the way to get there. **That inference is wrong in two separate ways at +once**, and every claim below is settled by a command rather than by reading. + +### Ask the gallery what exists, do not extrapolate from the family + +```bash +azd version +azd template list --output json | grep -c '"Azure-Samples/todo-' +``` + +Measured 2026-09-18: 303 templates, **zero** `todo-` entries. The family was archived 2026-09-04 and +left the gallery 2026-09-08, yet `azd init -t todo-csharp-sql` still initializes it and Microsoft +Learn still lists it, so users still arrive with it. `todo-csharp-cosmos-sql` is Cosmos DB for +NoSQL. **`todo-nodejs-sql` and `todo-python-sql` never existed.** + +### The archived web application blueprint for Azure SQL Database uses a password + +Read its infrastructure without cloning it: + +```bash +curl -s https://raw.githubusercontent.com/Azure-Samples/todo-csharp-sql/main/infra/app/db-avm.bicep \ + | grep -n -i "administratorLogin\|create user\|db_owner\|external provider" +``` + +Measured 2026-09-03: `administratorLogin`, `create user ${APPUSERNAME} with password`, +`alter role db_owner add member`, and zero hits for `external provider`. + +| It does | Not | +|---|---| +| Provisions the logical server with an `administratorLogin` and a password | Microsoft Entra-only authentication | +| Creates the application's database user with a password, from a deployment script | `CREATE USER ... FROM EXTERNAL PROVIDER` | +| Adds that user to `db_owner` | Any narrower role | +| Stores the connection string, user name and password included, in a vault | A passwordless connection string | + +**The template does create a managed identity, and that is what makes the misreading survive a +glance.** It is system-assigned to the API and it reads the vault. What the vault holds is a +connection string with a password in it. "It uses managed identity" is true of the vault and false +of the database. Converting the template is an infrastructure change, not an application one: the +server's authentication mode, the deployment script, the role and the connection string all move +together. + +### The firewall rule that survives choosing correctly + +```bash +for r in todo-csharp-sql:app/db-avm.bicep \ + functions-quickstart-dotnet-azd-sql:app/db.bicep \ + functions-quickstart-python-azd-sql:app/db.bicep \ + functions-quickstart-typescript-azd-sql:app/db.bicep; do + printf '%-42s ' "${r%%:*}" + curl -s "https://raw.githubusercontent.com/Azure-Samples/${r%%:*}/main/infra/${r##*:}" \ + | grep -c "startIpAddress: '0.0.0.1'" +done +``` + +**Four of four**, measured 2026-09-03: `1` on every line. All four build the logical server from +the Azure Verified Module and all four ship a rule named `Azure Services` running `0.0.0.1` to +`255.255.255.254`, **every address a client can present**. Three are Entra-only with a managed +identity and are otherwise the ones to copy, so picking the better template fixes the password and +leaves the firewall open. + +**The name points at a different rule.** The Azure CLI's own help for `--start-ip-address` says to +use `0.0.0.0` to represent all Azure-internal IP addresses, and Microsoft Learn records the Allow +Azure services special case as a rule whose start and end address are both `0.0.0.0`. A rule from +`0.0.0.1` to `255.255.255.254` is not a narrower version of it. + +**Say this to the user, then let them decide.** These are Microsoft-published samples and are not +ours to edit on someone's behalf. Name the rule out loud and offer to narrow it: + +```bash +az sql server firewall-rule list -g -s -o table +az sql server firewall-rule delete -g -s -n "Azure Services" + +# the documented Allow Azure services case, if Azure-internal traffic is what was meant +az sql server firewall-rule create -g -s -n AllowAzureServices \ + --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 + +# and the developer machine, if it connects directly +az sql server firewall-rule create -g -s -n dev-box \ + --start-ip-address --end-ip-address +``` + +**Narrow it after the first successful deployment, not before.** The post-provision hook runs from +a machine that then needs a rule of its own, and without one it fails as a timeout rather than as a +permission error. The three Functions quickstarts ship `infra/scripts/addclientip.ps1` for exactly +this. `provision-azure-sql-db` owns authoring firewall rules; this skill owns catching an inherited +one. + +### On any other template, run the same two greps before running anything + +```bash +grep -ri "external provider" ./infra | wc -l # 0 means the database user is password-based +grep -rn "IpAddress" ./infra # read the range, not the rule's name +azd provision --preview # what would be created, before it is +``` + +Open [references/azd-and-azure-sql.md](references/azd-and-azure-sql.md) **before** choosing a +template or writing the hook: it carries the per-template evidence, which templates are +Microsoft-published rather than community ones, and the hook wiring end to end. + +**What to copy instead.** The three serverless quickstarts are the pattern worth taking: an +Entra-only server, the deploying user as administrator, a user-assigned managed identity for the +application, and the grant run from a `postprovision` hook against the user database. Their roles +are `db_datareader`, `db_datawriter` and `db_ddladmin`, still wider than most applications need; +`least-privilege-database-roles` owns where to land. + +## The seam in the middle, which the good templates exist to close + +**The deployment tool provisions Azure resources and deploys code. A database user is neither.** The +application's identity means nothing to the database until somebody runs +`CREATE USER ... FROM EXTERNAL PROVIDER` and grants it roles, inside the database, from a connection +authenticated with Microsoft Entra ID. Until then the run goes green and the application fails at +its first query with a login failure that reads as a bad credential. + +`entra-id-auth` owns that statement, its clauses and its error codes. **Open it when you are about +to write the grant.** This skill owns where the grant goes in the sequence and how it gets run. + +## The four commands, and what each one actually does + +| Command | What it does | What it does not do | +|---|---|---| +| `init` | Sets up `azure.yaml`, an `infra` directory, an environment under `.azure`, from a template or from existing code | Create anything in Azure | +| `provision` | Creates the Azure resources and writes the template's outputs into the environment | Deploy code. Create anything inside the database | +| `deploy` | Pushes built application code to resources that already exist | Create resources. A deploy before a provision has nothing to deploy to | +| `up` | Runs packaging, provisioning and deployment in one command | Anything the individual steps do not do. It is a convenience, not extra behaviour | + +Keeping the two halves apart is how you debug them. Build the artifact yourself and hand it over: + +```bash +dotnet publish ./src/api -c Release -o ./artifacts/api +azd deploy api --from-package ./artifacts/api +``` + +**Do not depend on the internal ordering of `up`.** `azd up --help` calls it package, provision and +deploy; the hooks reference on Microsoft Learn calls it restore, provision and deploy. What is +stable is that provisioning happens before deployment and that `postprovision` runs after resources +are created. Pin the order explicitly if the project needs it, in the shape the command's own help +gives: + +```yaml +# azure.yaml +workflows: + up: + - azd: provision + - azd: package --all + - azd: deploy --all +``` + +## Step 1: know where the infrastructure is coming from + +`azure.yaml` at the project root is the only required file, mapping each service to a host and a +source directory: + +```yaml +name: my-project +services: + api: + project: ./src/api + language: csharp + host: appservice +``` + +Provisioning reads Bicep from `./infra` with `main.bicep` as the entry point, both overridable +under an `infra` key, `path` and `module`. + +**That Bicep has to come from somewhere, and there are only two real sources**: a template, read +first on the terms above, or hand-written from Azure Verified Modules. There is no third option, +and this is where an agent invents one: + +> **The compose feature cannot create an Azure SQL Database.** Its database resource types are +> Azure Cosmos DB, Azure Cosmos DB for MongoDB, Azure Cosmos DB for PostgreSQL, Azure Cache for +> Redis and Azure Database for MySQL. There is no Azure SQL Database entry, so `azd add` will not +> produce one, and `azd infra generate` only writes out what compose already knows. + +Creating the server, the database and the firewall rule with the Azure CLI instead belongs to +`provision-azure-sql-db`. + +## Step 2: put the grant between provisioning and deployment + +The database user has to exist after the identity and the database do, and before the application +serves its first request. There is exactly one place that is true, and it is the `postprovision` +hook, registered at the project root rather than inside a service: + +```yaml +# azure.yaml +hooks: + postprovision: + posix: + shell: sh + run: ./infra/scripts/configure-database.sh + continueOnError: false + interactive: false + windows: + shell: pwsh + run: ./infra/scripts/configure-database.ps1 + continueOnError: false + interactive: false +``` + +Inside the script, read what provisioning wrote and authenticate as the identity that provisioned: + +```bash +eval "$(azd env get-values | sed 's/^/export /')" +sqlcmd -S "$AZURE_SQL_SERVER_NAME" -d "$AZURE_SQL_DATABASE_NAME" \ + --authentication-method ActiveDirectoryAzureDeveloperCli \ + -i ./infra/scripts/grant-app-identity.sql +``` + +Develop that hook on its own, without reprovisioning: + +```bash +azd hooks run postprovision +``` + +Four things make this work, and each fails quietly if it is missing. + +1. **The infrastructure outputs the identity's name and principal id**, because a hook is a shell + script and can only see what provisioning wrote into the environment. +2. **The hook reads those outputs** rather than guessing resource names. +3. **`continueOnError` stays false.** Setting it true turns the one step that silently matters into + a step that silently does not happen. +4. **Whoever runs the hook is administrator of the logical server, authenticated with Microsoft + Entra ID.** `-G` alone falls back to `ActiveDirectoryDefault`, which can pick a different + signed-in identity than the one that just provisioned; naming the method is what pins it. + +## Step 3: get the connection string to the application + +**An environment value is not an application setting.** The value really is set and really does show +up locally, which is why this reads as success. + +| | Where it lives | What reads it | +|---|---|---| +| Environment value | `.azure//.env` in the project | Provisioning inputs, hooks, and the local command session | +| Application setting | The deployed application's configuration in Azure | The running application | + +The infrastructure has to write it as an application setting, which in Bicep means passing it into +the hosting module: + +```bicep +appSettings: { + AZURE_SQL_CONNECTION_STRING: 'Server=${sqlServer.outputs.fullyQualifiedDomainName}; Authentication=Active Directory Default; Database=${databaseName}; User Id=${apiIdentity.outputs.clientId}' +} +``` + +That is the shape Microsoft Learn gives for a function app reaching Azure SQL Database with a +managed identity. There is no password and no user name, which is the whole point. For a +user-assigned identity the **client** id has to be named, because the default credential otherwise +cannot choose between the identities attached to the host; for a system-assigned one, `User Id` is +omitted instead. The client id is not the principal id the grant uses. + +**Open `connect-to-azure-sql` when the driver is not `Microsoft.Data.SqlClient`**: the keyword +spellings differ per driver and are not guessable. + +## Step 4: know what teardown does and does not remove + +```bash +azd down # prompts for confirmation +azd down --purge # also permanently deletes resources that are soft-deleted by default +azd down --force --purge # no confirmation at all +``` + +It deletes the Azure resources for the environment, not local project files and not the +environment's stored values, so `.azure//.env` survives holding outputs that now name +resources that no longer exist. **The database goes with everything else** and its data is not +backed up by this command. **A database the project did not create is not deleted**, and the +database user created for the application identity survives inside it, pointing at a principal that +no longer exists. + +## Check it worked + +A green run is not the check. Four things, in this order, and the last one is the only one that +settles it. + +```bash +azd env get-values | grep -i sql # provisioning wrote the server and database names +azd show # the resources azd believes it owns +az sql server ad-admin list -g -s -o table +az sql server firewall-rule list -g -s -o table +az webapp config appsettings list -g -n \ + --query "[?name=='AZURE_SQL_CONNECTION_STRING'].name" -o tsv +``` + +Expect a server and database name in the environment, a named Entra administrator, no rule wider +than you chose, and the setting present on the deployed application. An empty result from the last +command is the Step 3 failure: the value exists locally and never reached Azure. + +Then confirm the database user exists, connected to the **user database** and not `master`: + +```sql +SELECT p.name, p.authentication_type_desc, r.name AS role_name +FROM sys.database_principals AS p +LEFT JOIN sys.database_role_members AS m ON m.member_principal_id = p.principal_id +LEFT JOIN sys.database_principals AS r ON r.principal_id = m.role_principal_id +WHERE p.name = ''; +``` + +Expect at least one row, `authentication_type_desc` reading `EXTERNAL`, and named roles that are not +`db_owner`. **Zero rows is the seam**: the grant did not run, or it ran against `master`. Finish by +calling an application endpoint that reads the database, because only the application proves its own +identity works. + +## Under an agent, prompts are failures + +`azd` documents `--no-prompt` on every command as automatically enabled when it detects a CI/CD or +AI-agent environment, failing if any required value cannot be resolved (`AZD_NON_INTERACTIVE=false` +opts out). So under an agent, any value that would have been prompted for is a hard failure. Set +them before running anything: + +```bash +azd env new +azd env set AZURE_SUBSCRIPTION_ID +azd env set AZURE_LOCATION +``` + +Hooks are the other half, and the two Learn pages disagree: the hooks reference says hooks run in +interactive mode by default, the schema reference says `interactive` defaults to false. Set it +explicitly on any hook that runs unattended rather than relying on either. + +## Where this stops + +| The request | The owner | +|---|---| +| Create the server, database and firewall rule with the Azure CLI | `provision-azure-sql-db` | +| The database user, the grant, the driver keywords, the error codes | `entra-id-auth` | +| Retry, pooling and encryption doctrine | `connect-to-azure-sql` | +| A pipeline that deploys on push or on merge | `github-actions-for-sql` | +| Getting the schema into the database | `schema-migrations-safely` | +| What roles to leave the application identity holding | `least-privilege-database-roles` | +| Sequencing a whole new project, before any of this | `build-app-on-azure-sql` | + +**The boundary with the pipeline is a real one, not a filing decision.** Deploying from a laptop +makes the developer administrator of the logical server, so the developer can run the grant. +Deploying from a pipeline makes the pipeline's identity administrator instead, and a developer then +cannot. `azd pipeline config` is the handoff point. + +## Do not + +- Do not treat a first-party template's `infra` directory as reviewed, and do not call one + passwordless because a managed identity appears in it. Check what that identity authenticates to. +- Do not accept a firewall rule because its name says Azure services. Read the range, and expect the + wide one even in the templates that get identity right. +- Do not extrapolate template names from the family, and do not report a deployment as working + because the tool reported success. +- Do not put the grant in a hook that runs after deployment, and do not set `continueOnError` true + on it. The application can serve a request before it runs. +- Do not set a connection string as an environment value and expect the deployed application to read + it, and do not leave `TrustServerCertificate` on in one copied out of a sample. +- Do not assume a database, server or resource name in a hook. Read it from the environment values + that provisioning wrote. +- Do not restate the `CREATE USER` statement, the role grants or the driver keyword spellings here. + `entra-id-auth` owns them, and two skills stating the same syntax is how they start disagreeing. + +## References + +- [references/azd-and-azure-sql.md](references/azd-and-azure-sql.md): open it before choosing a + template, before writing the post-provision hook, or when a claim above disagrees with what you + are seeing. It carries the per-template evidence, which templates are Microsoft-published, the + naming traps, and the hook wiring from infrastructure output to grant. +- `azd template list` and the community gallery at `azure.github.io/awesome-azd` are the only + current statement of what templates exist. List them rather than recalling them. +- [Hooks reference](https://learn.microsoft.com/azure/developer/azure-developer-cli/azd-extensibility) + and [azure.yaml schema](https://learn.microsoft.com/azure/developer/azure-developer-cli/azd-schema), + when a hook name or option is in question; + [environment variables](https://learn.microsoft.com/azure/developer/azure-developer-cli/manage-environment-variables) + when a value is not arriving where it was expected; + [network access controls](https://learn.microsoft.com/azure/azure-sql/database/network-access-controls-overview) + when a firewall rule needs justifying. diff --git a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/references/azd-and-azure-sql.md b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/references/azd-and-azure-sql.md new file mode 100644 index 0000000..6414ebb --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/references/azd-and-azure-sql.md @@ -0,0 +1,207 @@ +# Templates, and the post-provision grant wiring + +## Contents + +- [Templates that actually exist for Azure SQL Database](#templates-that-actually-exist-for-azure-sql-database) +- [The naming trap in the sample family](#the-naming-trap-in-the-sample-family) +- [What the first-party templates really do about identity](#what-the-first-party-templates-really-do-about-identity) +- [The firewall rule that survives choosing correctly](#the-firewall-rule-that-survives-choosing-correctly) +- [The post-provision grant, end to end](#the-post-provision-grant-end-to-end) +- [Failure table for the wiring](#failure-table-for-the-wiring) + +## Templates that actually exist for Azure SQL Database + +Verified 2026-09-03 against the Azure Developer CLI 1.32.0 by listing the gallery, which returned +315 entries, 17 of them tagged `azuresql`. Re-listed 2026-09-18: 303 entries, and the `todo-` family +gone. Names, tags and repositories move, so confirm before generating a project: + +```bash +azd template list -f azuresql --output json | grep -B2 '"msft"' +``` + +The Microsoft-published ones that use Azure SQL Database, as listed 2026-09-18: + +| Repository | What it is | +|---|---| +| `Azure-Samples/functions-quickstart-dotnet-azd-sql` | Serverless, triggers and bindings, .NET | +| `Azure-Samples/functions-quickstart-python-azd-sql` | The same, Python | +| `Azure-Samples/functions-quickstart-typescript-azd-sql` | The same, TypeScript | +| `Azure-Samples/azure-sql-db-session-recommender-v2` | Vector search and retrieval over Azure SQL Database | +| `Azure-Samples/nlp-sql-in-a-box` | Natural language over Azure SQL Database | + +**The tag is not a reliable filter in either direction**, measured 2026-09-03. +`Azure-Samples/blazor-azure-sql-vector-search` is Microsoft-published, uses Azure SQL Database, and +carries only `msft` and `ai`, so the filter misses it. `Azure-Samples/spring-petclinic-java-mysql` +carries `azuresql` and is MySQL by name. Read the repository, not the tag. + +Anything else in that filtered list is community-published rather than Microsoft-published, which is +worth saying to a user before it becomes the starting point for their project. + +**No Microsoft-published web application blueprint for Azure SQL Database is in the gallery any +more.** `Azure-Samples/todo-csharp-sql` (React front end, C# API) was archived on 2026-09-04, its +README now opening "This repository is archived and no longer maintained", and removed from the +gallery by Azure/awesome-azd pull request 1006 on 2026-09-08. It is not gone: `azd init -t +todo-csharp-sql` still initializes it on 1.32.0, measured 2026-09-18, and the Microsoft Learn page +"Full-stack deployment templates for Azure Developer CLI" still lists it. That is how users keep +arriving with it, so the evidence below still applies. + +The App Service tutorial on Microsoft Learn initializes `Azure-Samples/dotnet-app-service-sqldb-infra`, +which is not in the gallery. Read on 2026-09-18, its `infra/resources.bicep` sets +`administratorLoginPassword` and turns public network access off, so it is password-based too. + +## The naming trap in the sample family + +The archived blueprint samples are named `todo--`. Nine were in the gallery on +2026-09-03. **Only one uses Azure SQL Database**, and it is the C# one. Every other entry in the +family is MongoDB or Cosmos DB. + +Two specific mistakes follow from this, and both look reasonable: + +- **`todo-nodejs-sql` and `todo-python-sql` never existed.** GitHub returns 404 for both. An agent extrapolating from the family's + naming will produce a template name that fails at initialization. There is no first-party + blueprint sample pairing Node.js or Python with Azure SQL Database. The serverless quickstarts + above are the closest first-party starting points for those languages. +- **`todo-csharp-cosmos-sql` is not an Azure SQL Database template.** The `sql` in that name is + Cosmos DB for NoSQL, and its own listed description says the database is MongoDB. Selecting it + because the name contains `sql` produces a project with the wrong database entirely. + +## What the first-party templates really do about identity + +SKILL.md carries the headline. This is the per-template evidence behind it, gathered by reading each +`infra` directory on 2026-09-03, because the two generations of template differ completely. + +**The archived blueprint sample is password-based.** `infra/app/db-avm.bicep` provisions the logical server +with `administratorLogin` and a password. A deployment script then creates the application's +contained user with `create user ... with password` and runs `alter role db_owner add member`. The +string `FROM EXTERNAL PROVIDER` does not appear anywhere under `infra`. The application's managed +identity exists and is system-assigned, but its access policy is on the vault, and the secret the +vault holds is a connection string containing the user name and the password. Recommending this as +the passwordless path is wrong. Converting it moves the server's authentication mode, the deployment +script, the role and the connection string together, so it is an infrastructure change rather than +an application change. + +**The serverless quickstarts are identity-based.** They create the logical server with +`azureADOnlyAuthentication` on, make the deploying user the administrator, create a user-assigned +managed identity for the application, and run the grant from a post-provision hook. Microsoft ships +the remediation for the seam in these three as `infra/scripts/sql_add_uami_user.sql`. That is the +pattern worth copying. Their granted roles are `db_datareader`, `db_datawriter` and `db_ddladmin`, +narrower than the blueprint's `db_owner` and still wider than most applications need; +`least-privilege-database-roles` owns what to leave in place. + +## The firewall rule that survives choosing correctly + +| Template | Server module | Database authentication | Firewall rule shipped | +|---|---|---|---| +| `todo-csharp-sql` | Azure Verified Module | Password, `db_owner` | `Azure Services`, `0.0.0.1` to `255.255.255.254` | +| `functions-quickstart-dotnet-azd-sql` | Azure Verified Module | Entra-only, managed identity | The same rule | +| `functions-quickstart-python-azd-sql` | Azure Verified Module | Entra-only, managed identity | The same rule | +| `functions-quickstart-typescript-azd-sql` | Azure Verified Module | Entra-only, managed identity | The same rule | +| `azure-sql-db-session-recommender-v2` | Hand-written | Entra-based | `0.0.0.0` to `0.0.0.0` | +| `blazor-azure-sql-vector-search` | Hand-written | | No rule in `infra` | +| `nlp-sql-in-a-box` | Hand-written | | The deployer's own address | + +The split follows the module rather than the identity story: every template that builds the logical +server from the Azure Verified Module ships the wide rule, including the three that are otherwise +the ones to copy. The range is not in the module's own documented examples, so it travels with the +samples rather than with the module. SKILL.md carries the rest of the argument and the commands +that narrow it. + +### Where it is, verified 2026-09-03 + +All four files were fetched from `main` and read on that date. SKILL.md carries the one-liner that +reproduces this. + +| Template | File | Condition | +|---|---|---| +| `todo-csharp-sql` | `infra/app/db-avm.bicep` | unconditional | +| `functions-quickstart-dotnet-azd-sql` | `infra/app/db.bicep` | only when `vnetEnabled` is false | +| `functions-quickstart-python-azd-sql` | `infra/app/db.bicep` | only when `vnetEnabled` is false | +| `functions-quickstart-typescript-azd-sql` | `infra/app/db.bicep` | only when `vnetEnabled` is false | + +The literal shipped in all four: + +```bicep +{ + name: 'Azure Services' + startIpAddress: '0.0.0.1' + endIpAddress: '255.255.255.254' +} +``` + +## The post-provision grant, end to end + +Four pieces, in order. The whole thing fails silently if any one is missing. + +**1. The infrastructure creates the identity and outputs what the hook needs.** A hook is a shell +script and can only see values that provisioning wrote into the environment: + +```bicep +output AZURE_SQL_SERVER_NAME string = sqlServer.outputs.fullyQualifiedDomainName +output AZURE_SQL_DATABASE_NAME string = databaseName +output APP_IDENTITY_NAME string = apiIdentity.outputs.name +output APP_IDENTITY_PRINCIPAL_ID string = apiIdentity.outputs.principalId +output APP_IDENTITY_CLIENT_ID string = apiIdentity.outputs.clientId +``` + +The principal id is what identifies the identity to the database **in the `FROM EXTERNAL PROVIDER` +and `WITH OBJECT_ID` forms used here**. The client id is what goes into the application's connection +string. They are different values and they are not interchangeable. + +The `WITH SID` form does not follow the same rule: for a service principal or a managed identity it +takes the **client** id, not the principal id. That form and the reasons to reach for it belong to +`entra-id-auth`, which carries the full table. Do not carry this section's rule across to it. + +**2. The infrastructure makes the deploying identity administrator of the logical server**, with +Microsoft Entra-only authentication on. Without this the hook has no way to authenticate as +something that is allowed to create a user from an external provider. + +**3. `azure.yaml` registers the hook at the project root**, not inside a service. A service hook is +scoped to that service's lifecycle; the grant is not. + +```yaml +hooks: + postprovision: + posix: + shell: sh + run: ./infra/scripts/configure-database.sh + continueOnError: false + windows: + shell: pwsh + run: ./infra/scripts/configure-database.ps1 + continueOnError: false +``` + +**4. The script reads the outputs and runs the statement** against the user database, authenticating +with Microsoft Entra ID: + +```bash +#!/usr/bin/env sh +set -eu + +# Read what provisioning wrote, rather than guessing resource names. +eval "$(azd env get-values | sed 's/^/export /')" + +# The statement, its required clauses and its error codes belong to entra-id-auth. +# Run it against the USER database, never master. +sqlcmd -S "$AZURE_SQL_SERVER_NAME" -d "$AZURE_SQL_DATABASE_NAME" \ + --authentication-method ActiveDirectoryAzureDeveloperCli \ + -v IDENTITY_NAME="$APP_IDENTITY_NAME" PRINCIPAL_ID="$APP_IDENTITY_PRINCIPAL_ID" \ + -i ./infra/scripts/grant-app-identity.sql +``` + +`--authentication-method` and `-v` are both in `sqlcmd -?` on 1.10.0. +`ActiveDirectoryAzureDeveloperCli` reuses the credential the deployment is already running under, +where `-G` alone falls back to `ActiveDirectoryDefault` and can pick a different signed-in identity. +Run the hook on its own while developing it with `azd hooks run postprovision`. + +## Failure table for the wiring + +| What is missing | What you see | +|---|---| +| The identity's principal id is not an output | The script reads an empty value and creates a user that resolves to nothing, or fails on a blank argument | +| The hook is registered on a service instead of the project root | It runs for that service's lifecycle, and never for a provision-only run | +| `continueOnError` is set true | The grant fails, the deployment stays green, the application fails at its first query | +| The deploying identity is not administrator of the logical server | The statement is refused, and the message points at permissions rather than at the deployment | +| The machine running the hook has no firewall rule | The script cannot reach the database at all, and the message reads as a timeout | +| The grant runs against `master` | It succeeds, and the application still cannot log in, because the user was created in the wrong database | +| The connection string names the principal id instead of the client id | The application's credential cannot select the identity, and the failure looks like a bad connection string | diff --git a/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/skill.spec.jsonc new file mode 100644 index 0000000..c92f412 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/deploy-app-to-azure/skill.spec.jsonc @@ -0,0 +1,204 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "deploy-app-to-azure", + "domain": "azure-patterns-and-deploy", + "value": [ + "corrects-a-wrong-assumption", + "changes-too-fast-to-memorize", + "multi-step-with-silent-failures" + ], + "correction": "Agents treat a first-party Azure sample's infrastructure as a reviewed baseline and spend their attention on the application code, so asked to reach Azure SQL Database without a password using the Azure Developer CLI they recommend the first-party template as the way to get there. `Azure-Samples/todo-csharp-sql`, the Microsoft-published blueprint pairing a web application with Azure SQL Database, was archived and dropped from the gallery in September 2026 yet still initializes with `azd init`, and it does not use an identity for the database at all: it provisions the logical server with an administrator login and a password, creates the application's database user with `create user ... with password` from a deployment script, and adds that user to `db_owner`, with `FROM EXTERNAL PROVIDER` appearing nowhere under its infrastructure directory. The template does create a managed identity, which is what lets the misreading survive a glance, but that identity authenticates to a vault and what the vault holds is a connection string with a password in it. The second half is worse, because it survives choosing correctly: all four Microsoft-published templates that build the logical server from the Azure Verified Module, including the three serverless quickstarts that are otherwise exemplary about identity, ship a firewall rule named `Azure Services` spanning `0.0.0.1` to `255.255.255.254`, which is every address a client can present, while the documented Allow Azure services rule uses `0.0.0.0` as both its start and its end address. So the identity defect is fixed by choosing a different template and the firewall defect is not fixed by choosing anything.", + "value_probe": { + "gate": "The answer reads the infrastructure the template generates rather than trusting it because it is first-party, and reaches both how the application actually authenticates to the database and which addresses the generated firewall rule really admits.", + "task": "My team lead wants our new web application and its database deployed to Azure with no password anywhere, and asked me to start from a Microsoft published template rather than writing the infrastructure myself. I have found a first-party sample that pairs a web application with a SQL database, and on the face of it that is what we want. Before I commit the team to it, review the choice: does starting there actually get us where he is asking to be, and is there anything in the infrastructure it generates that I should not merge as it stands?", + "states": [ + "A managed identity can be present in a template while the database connection still uses a password held in a vault.", + "The start and end addresses of a generated firewall rule have to be read, because a wide range admits every address a client can present.", + "The documented allowance for Azure services uses one specific address as both its start and its end." + ], + "refutes": [ + "The first-party sample pairing a web application with a SQL database authenticates to that database with an identity.", + "The presence of a managed identity in the template means the database connection is passwordless.", + "Choosing a better published template removes the need to review the firewall rule it generates.", + "A firewall rule named for allowing Azure services admits only Azure and needs no further check." + ] + }, + "posture": [ + "provision", + "execute", + "admin" + ], + "maturity": "draft", + "applies_to": [ + "azure-sql-db" + ], + "related": [ + "azure-sql", + "build-app-on-azure-sql", + "provision-azure-sql-db", + "entra-id-auth", + "connect-to-azure-sql", + "diagnose-connection-errors", + "github-actions-for-sql", + "schema-migrations-safely", + "least-privilege-database-roles", + "azure-functions-sql-bindings", + "cutover-and-connection-string-swap" + ], + "validation": { + "target": "both", + "setup": [ + "a subscription in which a resource group, a logical server and a database can be created", + "a signed-in Microsoft Entra identity that can be made administrator of the logical server", + "an application project with source code and no infrastructure directory yet", + "the template gallery reachable, so candidate templates can be listed and their infrastructure read before one is chosen" + ], + "assert": [ + "the source of the infrastructure is named, and where it is a template, what that template does about database authentication was read rather than assumed", + "no template is called passwordless on the strength of a managed identity appearing in it, without establishing what that identity authenticates to", + "every firewall rule is judged by its address range rather than by its name, and a rule spanning the public address space is narrowed, with a rule added for whatever runs the post-provision hook", + "the role held by the application's database user is named, and db_owner is not carried over from a sample unexamined", + "the plan distinguishes what provisioning does from what deploying does, and places the database-side grant between them rather than after both", + "the grant runs from a post-provision hook that cannot be skipped silently, authenticated with Microsoft Entra ID as administrator of the logical server", + "the connection string reaches the deployed application as an application setting written by the infrastructure, carries no password, names the user database rather than master, and keeps encryption on with certificate trust off", + "the plan states which identity is administrator of the logical server after provisioning, and whether that identity is the developer or a pipeline", + "the teardown step names what survives it: the local environment values, and any database or logical server the project did not create" + ], + "probes": [ + { + "where": "host", + "cmd": "azd version", + "expect": { + "succeeds": true + }, + "why": "Every command in this skill is an Azure Developer CLI command, so the tool being installed and able to print its build number with nothing provisioned is the precondition for all of them. Measured 2026-09-03, exit 0, version 1.32.0." + }, + { + "where": "host", + "cmd": "azd provision --help 2>&1 | grep -q -- '--preview'", + "expect": { + "succeeds": true + }, + "why": "The skill tells a reader to run azd provision --preview before anything is created, which is only safe advice while the flag exists. Sourced from the command's own help on 1.32.0, measured 2026-09-03. If it is renamed, the skill's one safe rehearsal step goes with it and this probe goes red first." + }, + { + "where": "host", + "cmd": "azd deploy --help 2>&1 | grep -q -- '--from-package'", + "expect": { + "succeeds": true + }, + "why": "The skill shows dotnet publish handing an artifact to azd deploy --from-package, which is how the provision and deploy halves are kept apart while debugging. Sourced from azd deploy --help on 1.32.0, measured 2026-09-03." + }, + { + "where": "host", + "cmd": "azd down --help 2>&1 | grep -q -- '--purge'", + "expect": { + "succeeds": true + }, + "why": "The teardown section states that --purge also removes resources that are soft-deleted by default. Sourced from azd down --help on 1.32.0, measured 2026-09-03." + }, + { + "where": "host", + "cmd": "azd up --help 2>&1 | grep -q 'AZD_NON_INTERACTIVE=false'", + "expect": { + "succeeds": true + }, + "why": "The agent-behaviour claim made checkable: azd's own help says --no-prompt is automatically enabled when it detects a CI/CD or AI-agent environment, and names AZD_NON_INTERACTIVE=false as the opt-out. Measured 2026-09-03 on 1.32.0. An earlier revision quoted an error message nobody had reproduced; this is the tool's own text instead." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' 2>&1 | grep -q 'ActiveDirectoryAzureDeveloperCli'", + "expect": { + "succeeds": true + }, + "why": "The post-provision hook authenticates with ActiveDirectoryAzureDeveloperCli so it reuses the credential the deployment is already running under, where -G alone falls back to ActiveDirectoryDefault and can pick a different signed-in identity. Sourced from sqlcmd's own flag list, measured 2026-09-03 on go-sqlcmd 1.10.0." + }, + { + "where": "host", + "cmd": "az sql server firewall-rule create --help 2>&1 | grep -q \"Use value '0.0.0.0' to represent all Azure-internal IP\"", + "expect": { + "succeeds": true + }, + "why": "A second source for the address the Allow Azure services rule actually uses, independent of Microsoft Learn: the Azure CLI's help for --start-ip-address says 0.0.0.0 represents all Azure-internal IP addresses. That is what makes 0.0.0.1 to 255.255.255.254 a different rule rather than a narrower one. Measured 2026-09-03 on Azure CLI 2.90.0. Help output only, nothing is created." + }, + { + "where": "host", + "cmd": "g=\"$(azd template list --output json 2>/dev/null)\" && test \"$(printf '%s\\n' \"$g\" | grep -c '\\\"repositoryPath\\\"')\" -gt 100 && test \"$(printf '%s\\n' \"$g\" | grep -c '\\\"Azure-Samples/todo-')\" = 0", + "expect": { + "succeeds": true + }, + "why": "The gallery claim made executable: the gallery answers with more than 100 templates, so an unreachable gallery cannot pass as an empty one, and none of them is an Azure-Samples todo- template. Measured 2026-09-18 on 1.32.0: 303 templates, zero todo- entries, after Azure/awesome-azd pull request 1006 removed the family on 2026-09-08. Until that date this probe asserted exactly two todo-*-sql entries and went red when they left. If any todo- template returns, the skill's statement that the family left the gallery is stale." + }, + { + "where": "host", + "cmd": "curl -sf https://raw.githubusercontent.com/Azure-Samples/todo-csharp-sql/main/README.md | grep -q 'This repository is archived' && test \"$(curl -s -o /dev/null -w '%{http_code}' https://github.com/Azure-Samples/todo-nodejs-sql)\" = 404", + "expect": { + "succeeds": true + }, + "why": "The other half, read from GitHub rather than the gallery. todo-csharp-sql is archived but still published, which is why the skill still reads its Bicep: its README opens with the archive warning, added 2026-09-04. todo-nodejs-sql, the name an agent extrapolates from the family, returns 404; measured 2026-09-18. The gallery test alone cannot say this any more, because every todo- name is now absent from it. If the archive warning goes, or todo-nodejs-sql appears, the skill needs rereading." + }, + { + "where": "host", + "cmd": "test \"$(for r in todo-csharp-sql:app/db-avm.bicep functions-quickstart-dotnet-azd-sql:app/db.bicep functions-quickstart-python-azd-sql:app/db.bicep functions-quickstart-typescript-azd-sql:app/db.bicep; do curl -sf \"https://raw.githubusercontent.com/Azure-Samples/${r%%:*}/main/infra/${r##*:}\"; done | grep -c \"startIpAddress: '0.0.0.1'\")\" = 4", + "expect": { + "succeeds": true + }, + "why": "The headline claim made executable: four of four Microsoft-published templates that build the logical server from the Azure Verified Module ship a firewall rule starting at 0.0.0.1, including the three that are otherwise exemplary about identity. Measured 2026-09-03 by fetching each file from main. A count below 4 means a sample was fixed and the skill should say so." + }, + { + "where": "host", + "cmd": "curl -sf https://raw.githubusercontent.com/Azure-Samples/todo-csharp-sql/main/infra/app/db-avm.bicep | grep -q 'alter role db_owner add member'", + "expect": { + "succeeds": true + }, + "why": "The blueprint sample's deployment script adds the application's database user to db_owner. Measured 2026-09-03 in infra/app/db-avm.bicep. This is what makes recommending the template a correction rather than a preference." + }, + { + "where": "host", + "cmd": "curl -sf https://raw.githubusercontent.com/Azure-Samples/todo-csharp-sql/main/infra/app/db-avm.bicep | grep -qi 'external provider' && exit 1 || exit 0", + "expect": { + "succeeds": true + }, + "why": "The other half of the same reading, deliberately the negative: FROM EXTERNAL PROVIDER appears nowhere in the file, so the database user is password-based whatever the managed identity in the template is doing. Measured 2026-09-03, zero hits. The day this fails, the template was converted and the correction is stale." + } + ] + }, + "triggering": { + "explicit": [ + "Use the deploy-app-to-azure skill", + "Deploy my app and its Azure SQL database to Azure with azd", + "Which azd template should I start from for a web app with Azure SQL Database, and is it passwordless", + "Set up azd for this project so I can run azd up", + "Add an azure.yaml and infra folder to this app so it deploys to Azure SQL Database" + ], + "implicit": [ + "which azd template should I start from for a web app with a SQL database", + "I started from the Microsoft azd sample for Azure SQL and it created a SQL admin password, I thought it was passwordless", + "is the firewall rule that this azd template put in my infra folder safe to leave as it is", + "get this working local app into Azure without putting a database password anywhere", + "azd up finished successfully but my app returns a 500 as soon as it touches the database", + "my deployment is green but the app cannot log in to the database", + "how do I take my app and its database to Azure from my laptop" + ], + "routing": [ + "what is the difference between azd provision, azd deploy and azd up", + "is there a todo-nodejs-sql template", + "where does the connection string for my deployed app come from if there is no password", + "how do I add a SQL database to my azd project", + "azd down deleted my resources, what is left behind", + "how do I get from a working local project to Azure without putting a password in the repo" + ], + "negative": [ + "create an Azure SQL database and open the firewall for my address", + "narrow the firewall rule on my Azure SQL server to just my office IP range", + "set up Microsoft Entra authentication for my Azure SQL database and create the database user", + "build a GitHub Actions workflow that deploys schema changes to Azure SQL", + "run my EF Core migrations against Azure SQL Database", + "what does error 40615 mean", + "add retry logic to my Azure SQL data access layer", + "scaffold an app against the local Azure SQL Database container", + "expose my Orders table as a REST endpoint", + "grant my application identity the least privilege it needs on the database" + ] + } +} diff --git a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql deleted file mode 120000 index c08c1e4..0000000 --- a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/github-actions-for-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/SKILL.md b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/SKILL.md new file mode 100644 index 0000000..969bcd1 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/SKILL.md @@ -0,0 +1,310 @@ +--- +name: github-actions-for-sql +description: >- + Ships schema changes to Azure SQL Database from a GitHub Actions workflow with + `azure/sql-action`: building the database project or publishing a prebuilt dacpac, federating + the workflow's token so no database password or client secret is stored, getting a runner with + a changing address through the server firewall, and gating the deployment on an environment. + Use when a user asks to "deploy my database project from GitHub Actions", "publish a dacpac on + merge", "set up OIDC login to Azure for my pipeline", "stop storing a SQL password in secrets", + or "require an approval before the schema deploys", and when a run fails with "no matching + federated identity record found for presented assertion subject", a login error naming + `auth-type`, or "unable to detect client IP address". sql-database-projects owns the project + and publish options, entra-id-auth the database user and its grant. +--- + +# Deploy schema to Azure SQL Database from GitHub Actions + +**This owns the pipeline: the workflow, the login, the firewall and the gate.** Not the project +(`sql-database-projects`), the sequencing of a schema change (`schema-migrations-safely`), the +database-side grant (`entra-id-auth`), or the application (`deploy-app-to-azure`). + +Checked 2026-09-03 against `azure/sql-action` at tag `v2.4`, commit `eb1f9a80`, still the newest +tag; SqlPackage 170.4.83.3; sqlcmd 1.10.0; Azure CLI 2.90.0. + +## The action is three commands. Run them yourself first + +| Input | Notes | +|---|---| +| `connection-string` | **required**. This is where the authentication method is chosen | +| `path` | **required**. `.sql`, `.dacpac` or `.sqlproj`. A glob is allowed and must match exactly one file | +| `action` | **required for `.dacpac` and `.sqlproj`**. `Publish`, `Script`, `DriftReport`, `DeployReport`, nothing else | +| `arguments` | appended verbatim to the deployment command line, or to the sqlcmd line for a `.sql` path | +| `build-arguments` | appended to `dotnet build` when the path is a project | +| `skip-firewall-check` | default `false` | +| `sqlpackage-path` | point at a SqlPackage the runner already carries | + +The action shells out. These are the three command lines it composes, read from `src/SqlUtils.ts` +and `src/AzureSqlAction.ts` at `v2.4`. A red run that reproduces on a laptop is not a pipeline +problem. + +```bash +sqlcmd -S .database.windows.net,1433 -d master \ + --authentication-method=ActiveDirectoryDefault \ + -Q "SELECT 'Validating connection from GitHub SQL Action'" +``` + +That is the firewall probe, not the deployment, and it runs against `master` first, the target +database second. Then the build, for a `.sqlproj` path only: + +```bash +dotnet build "./db/ShopDb.sqlproj" -p:NetCoreBuild=true +``` + +Then the deployment, and this is the whole argument list rather than an excerpt of one: + +```bash +sqlpackage /Action:Publish \ + /TargetConnectionString:"Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=True;" \ + /SourceFile:"./db/bin/Debug/ShopDb.dacpac" +``` + +`arguments:` is appended to that line, which is how `/p:BlockOnPossibleDataLoss=true` reaches the +deployment. Which properties each action accepts is `sqlpackage-import-export`; open it before +writing a `/p:` you have not seen in that build's own help output. Four things that shape is +quietly telling you: + +- **The output path is guessed, not asked for.** The action looks for + `/bin//.dacpac`, configuration defaulting to `Debug`, + so an output redirected inside the project file rather than through `build-arguments` leaves it + looking in the wrong place. +- **`Extract`, `Import` and `Export` are refused**, real SqlPackage actions though they are: + `Action Extract is invalid. Supported action types are: Publish, Script, DriftReport, or DeployReport.` +- **The runner ships sqlcmd but not the deployment tool.** The action downloads `go-sqlcmd` 1.6.0 + into the tool cache itself; nothing brings SqlPackage, and on Linux the action calls the bare + name on `PATH`. One step fixes it with no path override, because `~/.dotnet/tools` is already on + `PATH` on the hosted image: `dotnet tool install -g microsoft.sqlpackage`. +- **macOS runners are refused** for the dacpac and project paths: + `This action is not supported on a Mac environment.` + +## Authenticating without a stored password + +Two different secrets are being avoided. **A database password** goes away with an identity-based +keyword in the connection string: `Active Directory Default` picks up the login step's +credentials, where `Active Directory Password` and `Active Directory Service Principal` put a +secret back and a plain user and password puts the database password back. **A client secret** +goes away by federating the workflow's token, which needs `permissions: id-token: write` in the +workflow below, without which there is no token to exchange. + +## The correction: read the subject claim, do not compose it + +The credential matches on issuer, audience and subject. The subject is the one that goes wrong, +and composing it from the documented `repo:OWNER/REPO:ref:refs/heads/main` is the failure. Ask the +repository what prefix it presents instead: + +```bash +gh api repos/OWNER/REPO/actions/oidc/customization/sub +``` + +Measured 2026-09-03 across twelve repositories in one account: eleven answered with the plain +name-based prefix, and the twelfth, one that had been renamed, answered with numeric ids in it: + +```json +{"use_default":true,"use_immutable_subject":false,"sub_claim_prefix":"repo:OWNER@/REPO@"} +``` + +**Both flags say the default is in force and the shape is still different.** GitHub's OIDC +reference explains why: repositories created, renamed or transferred after 15 July 2026 get an +immutable default subject built from owner and repository ids, and `use_immutable_subject` reports +only the separate opt-in, not the shape. Learn covers the Azure side under +[immutable subjects](https://learn.microsoft.com/entra/workload-id/workload-identities-github-immutable-subjects). +So a template works on eleven repositories and silently fails on the twelfth, which is worse than +failing everywhere. + +Take `sub_claim_prefix` and append the job segment: `:ref:refs/heads/`, +`:ref:refs/tags/`, `:environment:`, or `:pull_request`. A job naming an environment +presents `:environment:` **in place of** the ref segment, not in addition, so the two jobs +below need two credentials. + +```bash +az identity federated-credential create \ + --name gh-deploy-production --identity-name -g \ + --issuer "https://token.actions.githubusercontent.com" \ + --subject ":environment:production" \ + --audiences "api://AzureADTokenExchange" +``` + +Copy the prefix rather than retyping it: Learn records that credential matching became +case-sensitive in September 2024, so a name typed in the wrong case fails with no other symptom. + +A flexible federated identity credential can cover several job shapes at once by wildcard-matching +`sub` alongside `repository_id` or `repository_owner_id`. Learn calls it preview, application +objects only, and says Azure CLI errors on it, while Azure CLI 2.90.0 here exposes +`--claims-matching-expression-value`. The two disagree, so measure before planning around it. + +### The error names the wrong thing + +A subject that does not match produces `AADSTS700213: No matching federated identity record found +for presented assertion subject ''`. **That line is the most useful thing in the run, +because it quotes the subject actually presented.** Copy it into the credential. The line right +after it is the misleading one: + +```text +##[error]Login failed with Error: The process '/usr/bin/az' failed with exit code 1. +Double check if the 'auth-type' is correct. +``` + +`auth-type` is an input of the login step and is not the problem. An agent reading only the last +error changes it, or reaches for a client secret to go green, throwing away the point of +federating. Read upward to the AADSTS line and read its number: a directory can also refuse on +policy grounds, and that refusal is not fixable in the subject at all. + +## The firewall problem, and why it disguises the last one + +A hosted runner's public address changes between runs, so a fixed rule is not an option. What the +action does instead: + +1. Runs the `sqlcmd` probe above, against `master` first and the target database second. +2. If that fails, it searches the **error text** for an IPv4 address and adds a server-level rule + named `ClientIPAddress_--_
` for exactly that address. +3. It deploys, then drops the rule in a `finally` block, including when the deployment failed. + +Two consequences follow, and the second wastes the most time. + +**It needs a role on the server resource.** Adding a rule is a control-plane call, so the login +step must have run and the identity must be able to write firewall rules on the logical server. +Without that, the deployment fails at a step that is not the deployment. + +**Any login failure is reported as a firewall failure.** If the probe fails for a reason other +than the firewall, the error carries no address, and the action raises +`Failed to add firewall rule. Unable to detect client IP Address. `. That +message is about the firewall. The cause is usually the credential. Read past the first sentence. + +The two alternatives both pair with `skip-firewall-check: true`: a self-hosted runner on a fixed +address, which costs infrastructure and leaves a standing rule, or one reaching a private +endpoint, which exposes nothing publicly and is the most to set up. Allowing all Azure services is +not on that list on purpose. It is not a narrow rule. + +## Environment approvals, and what they do not protect + +The gate holds the job that names it, and nothing else in the workflow: + +- **Another job reads the same repository secrets.** They are available to every job, so anything + reachable with them is reachable without waiting. Put the deployment credential in + **environment** secrets, so the gate and the credential cover the same ground. +- **The credential subject is the real binding.** A subject ending `:environment:production` + cannot be exchanged by a job that does not name that environment. That is enforcement, where + the approval is only a pause. +- **The rule may not exist at all.** A required-reviewer rule on a private repository whose plan + excludes it is refused with `Failed to create the environment protection rule.` The environment + is still created, the workflow still references it, and every job runs unblocked. Read the + protection rules back rather than assuming the environment implies them. + +## A workflow shape that holds together + +```yaml +name: deploy-schema +on: + push: + branches: [main] +permissions: + id-token: write + contents: read +jobs: + plan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: dotnet tool install -g microsoft.sqlpackage + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - uses: azure/sql-action@v2.4 + with: + connection-string: ${{ secrets.SQL_CONNECTION_STRING }} + path: ./db/ShopDb.sqlproj + action: DeployReport + deploy: + needs: plan + runs-on: ubuntu-latest + environment: production # this changes the subject claim + steps: + - uses: actions/checkout@v4 + - run: dotnet tool install -g microsoft.sqlpackage + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - uses: azure/sql-action@v2.4 + with: + connection-string: ${{ secrets.SQL_CONNECTION_STRING }} + path: ./db/ShopDb.sqlproj + action: Publish +``` + +Two jobs, two credentials, because they present different subjects: one ending +`:ref:refs/heads/main` and one ending `:environment:production`. The connection string holds no +password, so it is a secret only to keep the server name out of a public log. `DeployReport` +writes the planned changes and touches nothing, which is what the reviewer releasing the second +job should be reading. + +## Check it worked + +A green run means the action exited zero. It does not say the schema landed, nor that the firewall +was left as it was found. Three checks, none of which changes anything. + +```bash +gh run view --json conclusion,jobs --jq '.conclusion, [.jobs[] | {name, conclusion}]' +``` + +Expect `success` and every job present. A job at `waiting` is the gate working, not a failure, and +a deploy job that never appears is a `needs:` that never released. + +```bash +sqlpackage /Action:DeployReport /SourceFile:"./db/bin/Debug/ShopDb.dacpac" \ + /TargetConnectionString:"" /OutputPath:after.xml +test -f after.xml || { echo "no report was written"; exit 1; } +grep -c "`, `grep -c` printed `0`, and the exit code was +`1`. + +**Expect it to be slow, and do not read slow as hung.** On a Basic tier database, measured the same +day, publishing a single three-column table took **3 minutes 16 seconds** and the DeployReport +after it took **2 minutes 50 seconds**. DacFx compares the whole model, so the floor is set by the +tier rather than by the size of the change. While it runs, `sys.dm_exec_sessions` shows a +`DacFx Deploy` session, which is how to tell a working deployment from a stalled one. + +**One line in a successful publish reads like a failure.** Between `Updating database (Start)` and +`Creating Table`, a project built from the default template prints +`'QUERY_STORE=OFF' is not supported in this version of SQL Server.` The publish continues and exits +zero. It is not an error and there is nothing to fix. + +```bash +az sql server firewall-rule list -g -s --query "[].name" -o tsv +``` + +Expect no name beginning `ClientIPAddress_`. The action removes its own rule even when the +deployment failed, so a leftover means the job was cancelled rather than failed. Do not read the +date out of that name: it is built from `getMonth()`, zero-based, and `getDay()`, the weekday. + +## Do not + +- Do not compose the subject from a template. Read the prefix the repository reports and append + the job segment, and do not add an environment to a job without adding its credential too. +- Do not act on the login step's closing line about `auth-type`, or treat "unable to detect client + IP address" as networking. Both name the step that reported, not the step that failed. +- Do not fall back to a client secret or a database password to get a red run green. That is the + one thing the design was for. +- Do not open the firewall to all Azure services to avoid managing a rule. It is not narrow. +- Do not assume declaring an environment created a gate, or that a gate on one job protects + another job holding the same secrets. + +## References + +- [references/oidc-and-firewall.md](references/oidc-and-firewall.md) holds the token payloads for both job shapes, the failure messages in full, the credential commands for an application identity as well as a user-assigned one, and the action's steps in order. Open it when a login or a firewall step fails and the message on screen does not say which of the two broke. +- [GitHub's OIDC reference](https://docs.github.com/en/actions/reference/security/oidc) when the subject claim is in question; [SqlPackage Publish](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-publish) and [DeployReport](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-deploy-drift-report) when the deployment step's behaviour is. diff --git a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/references/oidc-and-firewall.md b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/references/oidc-and-firewall.md new file mode 100644 index 0000000..1396ed6 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/references/oidc-and-firewall.md @@ -0,0 +1,230 @@ +# Measured: token claims, login failures, and what the action does + +## Contents + +- [How these were measured](#how-these-were-measured) +- [The subject claim, both job shapes](#the-subject-claim-both-job-shapes) +- [Finding the prefix without running anything](#finding-the-prefix-without-running-anything) +- [The three failure messages, in full](#the-three-failure-messages-in-full) +- [Creating the credential](#creating-the-credential) +- [What the action does, step by step](#what-the-action-does-step-by-step) +- [What the hosted runner ships](#what-the-hosted-runner-ships) +- [Environment protection rules](#environment-protection-rules) + +## How these were measured + +On 2026-08-28, on a throwaway repository with a user-assigned identity holding federated +credentials, deleted afterwards along with its resource group. `azure/sql-action` behaviour was +read from the source at tag `v2.4`, commit `eb1f9a80`, released 2026-07-23. The subject prefix +survey below was re-run on 2026-09-03 across twelve repositories in one account. + +Three credentials were created in turn against the same workflow, and the workflow was re-run +after each: a branch-shaped subject, the documented environment-shaped subject, and the subject +the token actually carried. + +## The subject claim, both job shapes + +A job requests its own token and decodes the payload. No third-party action is involved: + +```bash +TOKEN=$(curl -sS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | jq -r .value) +PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) +case $(( ${#PAYLOAD} % 4 )) in 2) PAYLOAD="${PAYLOAD}==";; 3) PAYLOAD="${PAYLOAD}=";; esac +echo "$PAYLOAD" | tr '_-' '/+' | base64 -d | jq -r '"sub = " + .sub, "aud = " + .aud, "iss = " + .iss' +``` + +Results from one run, two jobs differing only by an `environment:` line: + +```text +sub = repo:OWNER@/REPO@:ref:refs/heads/main +aud = api://AzureADTokenExchange +iss = https://token.actions.githubusercontent.com + +sub = repo:OWNER@/REPO@:environment:production +``` + +The environment segment replaces the ref segment. `aud` and `iss` are stable and are the two +easy fields to get right. + +## Finding the prefix without running anything + +```bash +gh api repos/OWNER/REPO/actions/oidc/customization/sub +``` + +Measured 2026-09-03 across twelve repositories nobody had configured. Eleven answered with the +plain name-based prefix: + +```json +{"use_default":true,"use_immutable_subject":false,"sub_claim_prefix":"repo:OWNER/REPO"} +``` + +The twelfth, a repository that had been renamed, answered with numeric ids and the same two flags: + +```json +{"use_default":true,"use_immutable_subject":false,"sub_claim_prefix":"repo:OWNER@/REPO@"} +``` + +Neither flag distinguishes them. GitHub's OIDC reference gives the rule: repositories created, +renamed or transferred after 15 July 2026 use an immutable default subject built from the owner +and repository ids, older ones keep the name-based shape unless they opt in, and +`use_immutable_subject` reports only that opt-in. Microsoft Learn documents the Azure half at +[migrate GitHub Actions federated credentials to immutable subjects](https://learn.microsoft.com/entra/workload-id/workload-identities-github-immutable-subjects), +using the same `repo:contoso@5544123/payments-api@821093847:ref:refs/heads/main` shape. So the +answer is per repository and cannot be predicted from either flag. Take `sub_claim_prefix` and +append the job segment: `:ref:refs/heads/`, `:ref:refs/tags/`, +`:environment:`, or `:pull_request`. + +## The three failure messages, in full + +**A subject that does not match**, whether because the environment segment was not accounted for +or because the prefix was composed by hand: + +```text +##[error]AADSTS700213: No matching federated identity record found for presented assertion +subject 'repo:OWNER@/REPO@:environment:production'. Please note that matching is +done using a case-sensitive comparison. Check your federated identity credential Subject, +Audience, and Issuer against the presented assertion. +https://learn.microsoft.com/entra/workload-id/workload-identity-federation +Trace ID: Correlation ID: Timestamp: +``` + +The case-sensitivity sentence is not decoration. Learn's authentication change log records that +federated credential matching moved from case-insensitive to case-sensitive in September 2024, so +a repository or environment name retyped in the wrong case fails here with no other symptom. + +**The line the login step prints next**, which names an input that is not the problem: + +```text +##[error]Login failed with Error: The process '/usr/bin/az' failed with exit code 1. +Double check if the 'auth-type' is correct. +``` + +**A directory policy refusal**, which looks like the first message and is not. This one was +returned once the subject matched, by a directory that requires an extra claim on tokens from +this issuer: + +```text +##[error]AADSTS7002381: Federated identity credentials issued by +'https://token.actions.githubusercontent.com/' for applications or managed identities registered +in this tenant must contain the enterprise claim with value '<...>' but actual value is ''. +``` + +The distinction matters. The first is fixed in the credential. The third cannot be fixed by +editing the subject at all, and it is also useful evidence: reaching it proves the subject +matched, because the subject is checked first. + +## Creating the credential + +For a user-assigned identity, which needs no application registration: + +```bash +az identity federated-credential create \ + --name gh-deploy-production \ + --identity-name \ + -g \ + --issuer "https://token.actions.githubusercontent.com" \ + --subject ":environment:production" \ + --audiences "api://AzureADTokenExchange" +``` + +For an application registration, the same three fields under a different command: + +```bash +az ad app federated-credential create --id --parameters @credential.json +``` + +One credential per job shape. A workflow whose plan job runs on the branch and whose deploy job +runs under an environment needs two of these. + +A flexible federated identity credential covers several shapes at once, matching `sub` with `*` +and `?` wildcards and, for GitHub, requiring `repository_id` or `repository_owner_id` alongside +it. Learn calls it preview, supported on application objects only, and says Azure CLI, Azure +PowerShell and Terraform all error on it, leaving Microsoft Graph, the portal, or `az rest`: + +```json +{"claimsMatchingExpression": {"value": "claims['sub'] matches 'repo:OWNER@/REPO@:*' and claims['repository_id'] eq ''", "languageVersion": 1}} +``` + +Azure CLI 2.90.0 on this host does expose `--claims-matching-expression-value`, marked preview and +mutually exclusive with `--subject`, so the tool and the documentation disagree about whether the +CLI can create one. That disagreement is unresolved here. Try it against your own tenant before +building a plan on either answer. + +## What the action does, step by step + +Read from `src/main.ts`, `src/SqlUtils.ts`, `src/FirewallManager.ts` and `src/AzureSqlAction.ts` +at `v2.4`: + +1. Downloads a pinned `go-sqlcmd` release into the runner tool cache and puts it on `PATH`. It + does not install the deployment tool. +2. Unless `skip-firewall-check` is `true`, connects with `sqlcmd` to `master`, then to the target + database if that fails. +3. On failure it searches the **error text** for an address. If it finds one, it obtains an + authorizer from the login step's session, resolves the logical server through the management + API, and adds a firewall rule whose start and end are that address. +4. If the error carries no address it throws + `Failed to add firewall rule. Unable to detect client IP Address. `. +5. For a project path, runs `dotnet build "" -p:NetCoreBuild=true `, then + looks for `/bin//.dacpac`, configuration + defaulting to `Debug`, which is why an output redirected inside the project file is not found. +6. Runs the deployment tool, appending anything in `arguments` verbatim. `DriftReport` is the one + action composed without `/SourceFile`, because it compares the target against its own snapshot. +7. Removes the firewall rule in a `finally` block, so it goes even when the deployment fails. + +Path resolution for the deployment tool: on Windows it collects every install it can find and +picks the highest version. On Linux it checks a global tool location and otherwise uses the bare +name on `PATH`. On macOS it throws `This action is not supported on a Mac environment.` + +## What the hosted runner ships + +Measured on `ubuntu-latest`, which resolved to `ubuntu-24.04`, image `20260823.283.1`: + +| Check | Result | +|---|---| +| `which sqlpackage` | not found | +| `~/.dotnet/tools` on `PATH` | yes, before the run installs anything | +| `dotnet --version` | `10.0.400` | +| installed SDKs | 8.0.x, 9.0.x and 10.0.x side by side | +| after `dotnet tool install -g microsoft.sqlpackage` | resolves at `/home/runner/.dotnet/tools/sqlpackage` | + +So one install step is enough, and it does not need a path override. + +## Environment protection rules + +Creating the environment and its rule, and reading back what actually exists: + +```bash +gh api -X PUT repos/OWNER/REPO/environments/production \ + --input - <<< '{"wait_timer":0,"reviewers":[{"type":"User","id":}]}' +``` + +On a private repository whose plan does not include the feature this is refused outright: + +```text +{"message":"Failed to create the environment protection rule. Please ensure the billing plan +supports the required reviewers protection rule.","status":"422"} +``` + +The environment itself is still created by the same call shape without the reviewer block, the +workflow referencing it is still valid, and jobs naming it run without waiting. So the absence of +a gate is silent from the workflow's side. Read the rules back: + +```bash +gh api repos/OWNER/REPO/environments/production --jq '[.protection_rules[].type]' +# ["required_reviewers"] +``` + +Releasing a waiting job, which is what a reviewer does: + +```bash +gh api repos/OWNER/REPO/actions/runs//pending_deployments \ + --jq '.[] | {env: .environment.name, can_approve: .current_user_can_approve}' +gh api -X POST repos/OWNER/REPO/actions/runs//pending_deployments \ + -f state=approved -f comment="reviewed" -F "environment_ids[]=" +``` + +Observed during one run with the rule in place: the job with no environment reported `success` +while the two jobs declaring the environment reported `waiting`, and both moved only after the +approval was posted. diff --git a/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/skill.spec.jsonc new file mode 100644 index 0000000..8251652 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/github-actions-for-sql/skill.spec.jsonc @@ -0,0 +1,152 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "github-actions-for-sql", + "domain": "schema-change-and-cicd", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize", "multi-step-with-silent-failures"], + "correction": "Wiring a workflow to deploy schema to Azure SQL Database, an agent composes the federated credential subject from the documented template `repo:OWNER/REPO:ref:refs/heads/main`, adjusting the job segment for the environment it just added. No amount of care with that segment helps, because the subject the platform actually presents is no longer the documented shape at all: the default prefix now carries numeric owner and repository ids, measured as `repo:OWNER@/REPO@:environment:` on a repository whose subject claim is reported as the default and not customised, which no hand-written credential will match. The prefix has to be read from the repository, not written. The identity provider rejects it with AADSTS700213, which does quote the presented subject verbatim, and the login step then prints `Login failed with Error: The process '/usr/bin/az' failed with exit code 1. Double check if the 'auth-type' is correct`, naming an input that is not the problem. The cost compounds one step later, because the deployment action opens the runner's firewall by parsing the client address out of a deliberately failed connection, so any authentication failure surfaces as `Failed to add firewall rule. Unable to detect client IP Address`, and a pipeline reports a firewall problem for an identity problem. The gate is also weaker than it reads: a required-reviewer rule is refused outright on a private repository whose plan does not include it, with the environment still created and the workflow still valid, and it only ever holds jobs that name the environment, so a second job in the same workflow reads the same repository secrets and deploys without waiting.", + "value_probe": { + "gate": "The subject registered for the credential is read from what the repository actually presents rather than composed by hand from a documented template, and the firewall message is identified as a consequence of the failed sign in rather than a second problem to fix.", + "task": "Our workflow deploys schema changes to Azure SQL Database and it worked until I added a deployment environment to the job. Now the sign in step fails and the log tells me to double check that the auth type is correct, and the step after it says it could not detect the client IP address so it could not add a firewall rule. I have re-created the credential in Azure twice, matching the branch and the environment name exactly as the documentation shows. What is wrong, and how can I verify the fix before I push again?", + "states": [ + "The subject the platform presents can carry numeric owner and repository ids, so it has to be read rather than written.", + "The identity provider's own rejection quotes the subject that was presented, which is where to read the real value.", + "An environment rule only holds jobs that name that environment, so another job in the same workflow deploys without waiting." + ], + "refutes": [ + "The subject to register is the documented template carrying the owner and repository names.", + "The message about not detecting the client IP address means the firewall or the runner's networking is the problem.", + "The auth type input named in the sign in failure is the input that needs correcting.", + "A required reviewer rule on the environment can be relied on to hold the whole workflow." + ] + }, + "posture": ["execute", "provision", "admin"], + "maturity": "ga", + "applies_to": ["azure-sql-db"], + "related": [ + "sql-database-projects", + "schema-migrations-safely", + "deploy-app-to-azure", + "entra-id-auth", + "provision-azure-sql-db", + "connect-to-azure-sql", + "azuresql-db-ci" + ], + "validation": { + "target": "both", + "setup": [ + "a repository that can run workflows, with a workflow able to request an identity token", + "an application identity or user-assigned identity in the directory that can hold a federated credential", + "a logical server and database already provisioned, since the workflow deploys schema and does not create infrastructure", + "an identity with a role on the logical server resource if the deployment step is expected to manage a firewall rule" + ], + "assert": [ + "the federated credential subject was taken from what the platform presents rather than composed from a documented template, and the way it was obtained is stated", + "the subject matches the job shape that will actually run, including the environment segment whenever the job names an environment", + "the workflow grants the identity-token permission at the job or workflow level, because without it the login step has no token to exchange", + "no database password, client secret or connection secret is stored in the repository, and the connection string carries an identity-based authentication keyword instead", + "the plan states how the runner reaches the server through the firewall, and the chosen answer is named rather than left to the action's default behaviour", + "an authentication failure is not reported to the user as a firewall failure, and the underlying message is read before the surface one", + "the deployment step is given a target file whose type matches what the step is asked to do, and the action input naming the operation is supplied wherever it is required", + "the runner has the deployment tool available, because the hosted image does not carry it", + "the environment gate is stated to hold only the jobs that name it, and any job outside it that can reach the same secrets is called out" + ], + "probes": [ + { + "where": "host", + "cmd": "sqlpackage /Action:Publish /help:True | grep -q '^/TargetConnectionString:$' && sqlpackage /Action:Publish /help:True | grep -q '^/SourceFile:$'", + "expect": { "succeeds": true }, + "why": "azure/sql-action does not have its own deployment engine: for Publish, Script and DeployReport it composes exactly '/Action: /TargetConnectionString:\"\" /SourceFile:\"\"' and shells out, read from src/AzureSqlAction.ts at tag v2.4. The skill tells a reader to run that same line locally before trusting the pipeline, so both parameters have to be real. Measured 2026-09-03 on SqlPackage 170.4.83.3, both present in the Publish help. If a build renames either one, the local reproduction in the body stops working and so does the action." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:DeployReport /help:True | grep -q '^/OutputPath:$'", + "expect": { "succeeds": true }, + "why": "The check-it-worked section reruns DeployReport against the deployed database and counts operations in the XML, and the file only exists if the path parameter is right. Measured 2026-09-03 on 170.4.83.3: DeployReport takes /OutputPath. Microsoft Learn's SQL projects automation page uses the same parameter for this action." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:DeployReport /help:True | grep -q '^/DeployReportPath' && exit 1 || exit 0", + "expect": { "succeeds": true }, + "why": "The negative half of the pair above, and the trap the skill names: /DeployReportPath is a Publish parameter and DeployReport does not accept it, so a report requested with the wrong flag is simply not written and the check silently measures nothing. Measured 2026-09-03 on 170.4.83.3: absent from the DeployReport help, present in the Publish help. This probe goes red the day the two actions converge, which would make the skill's warning wrong." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' | grep -q -- '--authentication-method'", + "expect": { "succeeds": true }, + "why": "Before deploying anything the action opens a sqlcmd connection to master purely to read the runner's address out of the failure, and for an identity-based connection string it builds that call with --authentication-method=ActiveDirectoryDefault, read from src/SqlUtils.ts at v2.4. The skill prints that command line for a reader to run locally. Measured 2026-09-03 against sqlcmd 1.10.0 on this host; the action ships go-sqlcmd 1.6.0 into the runner tool cache, so a flag that disappeared between those versions would break the reproduction rather than the pipeline." + }, + { + "where": "host", + "engine": true, + "cmd": "set -eu\nd=$(mktemp -d); trap 'rm -rf \"$d\"' EXIT\nmkdir -p \"$d/db\"\ncat > \"$d/db/ShopDb.sqlproj\" <<'PROJ'\n\n \n ShopDb\n Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider\n \n\nPROJ\ncat > \"$d/db/Product.sql\" <<'TBL'\nCREATE TABLE [dbo].[Product] ([ProductId] INT NOT NULL PRIMARY KEY, [Name] NVARCHAR(200) NOT NULL, [ListPrice] DECIMAL(10,2) NOT NULL);\nTBL\ndotnet build \"$d/db/ShopDb.sqlproj\" -p:NetCoreBuild=true >/dev/null\ntest -f \"$d/db/bin/Debug/ShopDb.dacpac\"\nsqlpackage /Action:Publish /TargetConnectionString:\"Server=tcp:localhost,${PROBE_PORT};Initial Catalog=${PROBE_DB};User ID=${PROBE_USER};Password=${PROBE_PASSWORD};Encrypt=True;TrustServerCertificate=True;\" /SourceFile:\"$d/db/bin/Debug/ShopDb.dacpac\" >/dev/null\nn=$(sqlcmd -S localhost,$PROBE_PORT -U \"$PROBE_USER\" -P \"$PROBE_PASSWORD\" -d \"$PROBE_DB\" -C -b -h -1 -W -Q \"SET NOCOUNT ON; SELECT COUNT(*) FROM sys.tables WHERE name = 'Product';\")\ntest \"$n\" = \"1\"", + "expect": { + "succeeds": true + }, + "why": "THE FIRST PROBE ON THIS SKILL THAT DEPLOYS ANYTHING. The four before it read help output: they prove the parameter names in the skill's command lines are real and prove nothing about whether the pipeline works. This runs the action's three steps in order against a live engine, using the exact argument list the skill prints: dotnet build the .sqlproj, then sqlpackage /Action:Publish /TargetConnectionString: /SourceFile:, then a query for the table. It also pins the output path the action GUESSES rather than asks for, bin//.dacpac, which the skill calls out as a trap. Red if the SDK moves that path, if the publish is refused, or if the table is not in sys.tables afterwards. A green run here is the claim 'this pipeline deploys schema' with something behind it." + }, + { + "where": "host", + "engine": true, + "cmd": "set -eu\nd=$(mktemp -d); trap 'rm -rf \"$d\"' EXIT\nmkdir -p \"$d/db\"\ncat > \"$d/db/ShopDb.sqlproj\" <<'PROJ'\n\n \n ShopDb\n Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider\n \n\nPROJ\ncat > \"$d/db/Product.sql\" <<'TBL'\nCREATE TABLE [dbo].[Product] ([ProductId] INT NOT NULL PRIMARY KEY, [Name] NVARCHAR(200) NOT NULL, [ListPrice] DECIMAL(10,2) NOT NULL);\nTBL\ndotnet build \"$d/db/ShopDb.sqlproj\" -p:NetCoreBuild=true >/dev/null\nsqlpackage /Action:DeployReport /SourceFile:\"$d/db/bin/Debug/ShopDb.dacpac\" /TargetConnectionString:\"Server=tcp:localhost,${PROBE_PORT};Initial Catalog=${PROBE_DB};User ID=${PROBE_USER};Password=${PROBE_PASSWORD};Encrypt=True;TrustServerCertificate=True;\" /OutputPath:\"$d/after.xml\" >/dev/null\ntest -f \"$d/after.xml\"\ntest \"$(grep -c ' \"$d/db/ShopDb.sqlproj\" <<'PROJ'\n\n \n ShopDb\n Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider\n \n\nPROJ\ncat > \"$d/db/Product.sql\" <<'TBL'\nCREATE TABLE [dbo].[Product] ([ProductId] INT NOT NULL PRIMARY KEY, [Name] NVARCHAR(200) NOT NULL);\nTBL\ndotnet build \"$d/db/ShopDb.sqlproj\" -p:NetCoreBuild=true >/dev/null\nsqlcmd -S localhost,$PROBE_PORT -U \"$PROBE_USER\" -P \"$PROBE_PASSWORD\" -d \"$PROBE_DB\" -C -b -Q \"INSERT INTO dbo.Product (ProductId, Name, ListPrice) VALUES (1, N'a', 9.99);\" >/dev/null\nif sqlpackage /Action:Publish /SourceFile:\"$d/db/bin/Debug/ShopDb.dacpac\" /TargetConnectionString:\"Server=tcp:localhost,${PROBE_PORT};Initial Catalog=${PROBE_DB};User ID=${PROBE_USER};Password=${PROBE_PASSWORD};Encrypt=True;TrustServerCertificate=True;\" /p:BlockOnPossibleDataLoss=true > \"$d/out.txt\" 2>&1; then\n echo \"the publish SUCCEEDED and it was supposed to be blocked\"; cat \"$d/out.txt\"; exit 1\nfi\ngrep -qi \"data loss\" \"$d/out.txt\"\nn=$(sqlcmd -S localhost,$PROBE_PORT -U \"$PROBE_USER\" -P \"$PROBE_PASSWORD\" -d \"$PROBE_DB\" -C -b -h -1 -W -Q \"SET NOCOUNT ON; SELECT COUNT(*) FROM sys.columns WHERE object_id = OBJECT_ID('dbo.Product') AND name = 'ListPrice';\")\ntest \"$n\" = \"1\"", + "expect": { + "succeeds": true + }, + "why": "/p:BlockOnPossibleDataLoss=true is the one deployment property the skill names, and arguments: is the input that carries it through to the command line. Nothing had ever checked that it blocks. The probe publishes a dacpac whose Product table has lost the ListPrice column, against a database where that column holds a row, and asserts three things: the publish exits non-zero, its output says data loss, and the column is STILL THERE afterwards. The last assertion is the one that matters. A refusal that had already dropped the column would be worse than no refusal, and an exit code alone cannot tell the two apart. Red if the property stops blocking, or if it blocks after the fact." + }, + { + "sql": "BEGIN TRY\n EXEC sp_executesql N'USE [master];';\n SELECT -1 AS unreached;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_NUMBER() AS n;\nEND CATCH", + "expect": { + "rows": [ + [ + 40508 + ] + ] + }, + "why": "Why the action opens TWO connections rather than one. The skill states the firewall probe runs 'against master first, the target database second', and the reason is that this engine refuses to switch database inside a session: USE raises Msg 40508. An agent that read that sentence as a stylistic choice would collapse the two calls into one connection with a USE and get a pipeline that fails after the login succeeds. The catch is narrow on purpose: it asserts the NUMBER, so a different failure, including a malformed batch, reports its own number and the probe goes red rather than passing on any error at all. Red if this engine ever allows USE." + } + ] + }, + "triggering": { + "explicit": [ + "Use the github-actions-for-sql skill", + "write a GitHub Actions workflow that deploys my database project to Azure SQL Database", + "set up federated credentials so my workflow logs in to Azure without a stored password", + "gate my database deployment on an environment approval", + "publish this dacpac from CI on merge to main" + ], + "implicit": [ + "my workflow logs in fine until I add an environment to the job, then it fails", + "no matching federated identity record found for presented assertion subject", + "the deploy step says it cannot detect the client IP address", + "how does the runner get through the Azure SQL firewall when its address changes every run", + "the pipeline cannot find sqlpackage on the runner", + "I do not want a SQL password sitting in repository secrets" + ], + "routing": [ + "which action deploys a sqlproj rather than a prebuilt dacpac", + "can the workflow produce a deployment script instead of deploying", + "what permissions does the workflow need to request a token", + "do environment approvals stop other jobs in the same workflow", + "why does the login error mention auth-type" + ], + "negative": [ + "set the target platform on my SQL database project so the build validates against Azure SQL Database", + "my dacpac publish reported success and the column data is gone", + "should this column change be done as expand and contract", + "deploy my web app and its database to Azure with azd", + "create the database user for my application identity and grant it roles", + "run integration tests against a local database container in CI", + "create an Azure SQL database and a logical server in my subscription", + "add retry logic to my data access layer" + ] + } +} diff --git a/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db deleted file mode 120000 index 0a66007..0000000 --- a/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db +++ /dev/null @@ -1 +0,0 @@ -../../../skills/provision-azure-sql-db \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/SKILL.md b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/SKILL.md new file mode 100644 index 0000000..c2c0dea --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/SKILL.md @@ -0,0 +1,198 @@ +--- +name: provision-azure-sql-db +description: >- + Creates an Azure SQL Database and returns a connection string that actually works, covering the + free offer and paid tiers, the firewall rule, and Microsoft Entra-only administration. Use when a + user asks to "create an Azure SQL database", "set up a free SQL database in Azure", "provision + SQL for this app", "give me a connection string", or when an application needs a cloud database + and none exists yet. Also use when a provisioning attempt succeeded but nothing can connect, + which is almost always the missing firewall rule. Covers how many free databases a subscription + actually gets and the exhaustion behavior value the CLI accepts, which is not the one its own help + text describes. +--- + +# Provision an Azure SQL Database + +Creating the database is the easy part. Three things go wrong: **nothing can reach it**, **the free +offer is configured with a value the CLI rejects**, and **the free offer runs out saying nothing**. + +Verified against Azure CLI **2.90.0** on 2026-09-03: every flag below was read from that CLI's own +`--help`, and the free offer numbers from the product documentation the same day. + +## The shape of a working provision + +Four commands, not one. A plan with fewer than four is incomplete. + +```bash +# 1. resource group +az group create -n -l + +# 2. logical server +az sql server create -g -n -l \ + --admin-user --admin-password + +# 3. database +az sql db create -g -s -n \ + --edition GeneralPurpose --compute-model Serverless --family Gen5 --capacity 2 + +# 4. FIREWALL. Separate command. Step 3 has no flag for this. +az sql server firewall-rule create -g -s -n allow-my-ip \ + --start-ip-address --end-ip-address +``` + +Hyperscale has its own plan and a one-way door in it: `provision-hyperscale`. + +## Gotcha: the firewall is not part of database creation + +`az sql db create` has **no firewall argument at all**. Confirm without provisioning anything: + +```bash +az sql db create --help | grep -ci firewall # 0 +``` + +A plan that stops after step 3 produces a database that exists, bills, and refuses every connection +with a login timeout. It is the most common way an agent-generated provision fails, because the +failure surfaces at connect time and looks like a credentials problem. + +To let other Azure services reach the server, use a rule from `0.0.0.0` to `0.0.0.0`. Both address +flags say so in their own help: "Use value '0.0.0.0' to represent all Azure-internal IP +addresses." + +```bash +az sql server firewall-rule create -g -s -n allow-azure-services \ + --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 +``` + +That is not an internet wildcard. `0.0.0.0` to `255.255.255.255` is what a wildcard would be, and +it is wrong. + +## Gotcha: the free offer's exhaustion value is not what the help text says + +```bash +az sql db create -g -s -n \ + -e GeneralPurpose -f Gen5 -c 2 --compute-model Serverless \ + --use-free-limit --free-limit-exhaustion-behavior AutoPause +``` + +`--free-limit-exhaustion-behavior` accepts exactly **`AutoPause`** and **`BillOverUsage`**. The +CLI's own help for that flag describes a value called `BillForUsage`. **That value does not exist**, +and passing it fails before any request reaches Azure: + +``` +ERROR: az sql db create: 'BillForUsage' is not a valid value for +'--free-limit-exhaustion-behavior'. Allowed values: AutoPause, BillOverUsage. +``` + +`AutoPause` makes the database inaccessible until the next calendar month and bills nothing. +`BillOverUsage` keeps it online, bills the overage, and **cannot be changed back**. Choose +`AutoPause` when the user says "free"; never pick `BillOverUsage` on their behalf. + +### How many free databases, and the limit + +**Up to 10 per subscription**, each with its own monthly **100,000 vCore seconds, 32 GB data and +32 GB backup storage**. The allowance is per database, unused seconds do not carry forward, and a +free database cannot be created by restoring an existing one. + +**The CLI help for `--use-free-limit` says "Allowed on one database in a subscription".** That is +the second wrong statement in this command's own help, and it is wrong in the direction that makes +an agent refuse work it could do. + +Neither number is a client-side gate. Measured with Azure Resource Graph on 2026-09-03: 47 +free-limit databases across two subscriptions, one holding 39. **When free capacity is refused the +create call returns a bare HTTP 500 with no quota message**, documented nowhere on Microsoft Learn. +Read that 500 as "no free slot": drop `--use-free-limit` for an ordinary billed database, or free +one and wait, because **a deleted slot takes up to an hour** to come back. + +## Microsoft Entra-only administration + +Passwordless is the better default, and it is set on the **server**, not the database. It needs +three flags together, and the SID is the object id of the principal rather than its name: + +```bash +az sql server create -g -n -l \ + --enable-ad-only-auth \ + --external-admin-principal-type User \ + --external-admin-name "" \ + --external-admin-sid $(az ad signed-in-user show --query id -o tsv) +``` + +With `--enable-ad-only-auth`, `--admin-user` and `--admin-password` are not used and SQL +authentication is off. Do not pass both models and hope one wins. + +Those flags set the administrator and nothing else. **There is no `az` command for any other +principal**, so the plan needs a fifth step that is T-SQL, run as the administrator because only it +can create the first one: `CREATE LOGIN [] FROM EXTERNAL PROVIDER` in virtual `master`, which +is what server roles attach to, or `CREATE USER [] FROM EXTERNAL PROVIDER`, which reaches one +database only. + +Stop there: choosing between them, granting, and diagnosing a login that still fails are all +`entra-id-auth`. Open +[Microsoft Entra server principals](https://learn.microsoft.com/azure/azure-sql/database/authentication-azure-ad-logins) +when you need the exact syntax rather than recalling it. + +## The connection string + +Ask the CLI rather than assembling one by hand. It runs offline and creates nothing: + +```bash +az sql db show-connection-string -c ado.net -s -n +``` + +- **`Initial Catalog` names the user database, not `master`.** A string left pointing at `master` + connects and then fails on the first real query. +- **`Encrypt=true`** stays. It is the default and it is not negotiable here. +- **`-a` cannot produce the production string.** It offers only `SqlPassword`, the default, plus + `ADPassword` and `ADIntegrated`. A service running in Azure should use a managed identity, which + Microsoft Learn calls the recommended method for programmatic access to SQL, so edit in + `Authentication="Active Directory Managed Identity"`. Never hand back the `User ID` and + `Password` template with a real password filled in. + +## Check it worked + +Three checks. The first two need no credential and no engine. + +**One: it exists, and it is what was asked for.** + +```bash +az sql db show -g -s -n -o yamlc \ + --query "{slo:currentServiceObjectiveName,status:status,free:useFreeLimit}" +``` + +Expect `slo: GP_S_Gen5_2` for the plan above, and `free: true` only if the free offer was asked +for. `status: Paused` is **not** a failure: serverless auto-pauses, and 46 of the 47 free-limit +databases visible on 2026-09-03 read `Paused`. + +**Two: the firewall rule is on the server.** + +```bash +az sql server firewall-rule list -g -s -o table +``` + +Expect at least one row. An empty list is the failure this skill exists for. + +**Three: you can reach it, and it is Azure SQL Database.** + +```bash +export SQLCMDPASSWORD='' +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 \ + -Q "SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS engine_edition, DB_NAME() AS db;" +echo "exit $?" +``` + +Expect exit `0`, `engine_edition` of **`5`**, the value `SERVERPROPERTY` documents for Azure SQL +Database, and `db` equal to your database and never `master`. + +Read the two failures apart. A login **timeout** with no message number is check two: the firewall +rule is missing. **`Msg 40613`**, the database is not currently available, is a paused serverless +database resuming and usually clears inside a minute, so retry. The application-side retry policy is +`connect-to-azure-sql`'s; any other number is `diagnose-connection-errors`'. + +## Checklist before reporting success + +- [ ] A firewall rule exists and `firewall-rule list` returns it +- [ ] `SERVERPROPERTY('EngineEdition')` returned 5 over the connection string being handed back +- [ ] The connection string names the user database and carries no password +- [ ] If free was requested, `--use-free-limit` is set and the behaviour is `AutoPause` or + `BillOverUsage`, never `BillForUsage` +- [ ] If passwordless was requested, the server has `--enable-ad-only-auth` and no SQL admin +- [ ] Any identity beyond the administrator has a named principal and someone to create it diff --git a/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/skill.spec.jsonc new file mode 100644 index 0000000..a473e3e --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-azure-sql-db/skill.spec.jsonc @@ -0,0 +1,115 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "provision-azure-sql-db", + "domain": "provisioning-and-local-dev", + "value": ["not-in-training-data", "changes-too-fast-to-memorize"], + "correction": "Asked to create an Azure SQL database, an agent writes a plausible az sql db create and stops there, producing a database nothing can reach: it omits the firewall rule, which is a separate command that az sql db create has no flag for, and it fills the free offer's exhaustion behavior from the documented value BillForUsage, which the CLI rejects because the only accepted values are AutoPause and BillOverUsage. The third thing it gets wrong is a release behind rather than invented: Microsoft Entra server principals, Entra logins created with CREATE LOGIN FROM EXTERNAL PROVIDER in the virtual master database, are generally available on Azure SQL Database, so a plan that offers a contained database user as the only way to give an identity access misses the principal that server roles attach to, and neither shape is an az command.", + "value_probe": { + "gate": "The result is a database that a client on the author's own machine can actually reach, so admitting that client address is a step of its own with its own command, rather than something the database creation is assumed to cover.", + "task": "Please give me the exact commands to stand up a new Azure SQL Database on the free monthly amount, and a connection string I can use from my laptop straight afterwards. When the free amount for the month is used up I want the database to stop rather than start charging us. Later I also need to give our deployment identity access to it, so tell me what that takes as well. Copy and paste ready, and say what each command is for.", + "states": [ + "Admitting a client address is a separate command, with no flag for it on the database creation command.", + "Microsoft Entra logins created in the virtual master database are available on this service and are what server roles attach to.", + "The accepted values for the free offer's exhaustion behaviour have to be read from the tool rather than from the documentation." + ], + "refutes": [ + "Creating the database with the Azure CLI is by itself enough for a client on the developer's machine to connect to it.", + "BillForUsage is an accepted value for the behaviour when the free monthly amount is exhausted.", + "A contained database user is the only way to give a Microsoft Entra identity access to the database.", + "Command line help text can be trusted for the accepted values of a flag." + ] + }, + "posture": ["read", "write", "provision"], + "maturity": "ga", + "applies_to": ["azure-sql-db"], + "related": ["connect-to-azure-sql", "entra-id-auth", "provision-hyperscale", "diagnose-connection-errors"], + "validation": { + "target": "both", + "setup": [ + "Azure CLI 2.90.0 or later on the host, signed in; every cmd probe reads help or renders a template locally and none of them creates an Azure resource", + "a scratch database on the probe target for the one sql probe" + ], + "assert": [ + "a firewall rule exists, created by a separate command, and the plan verifies it with firewall-rule list rather than assuming it", + "the free offer limit is stated as 10 per subscription, not 1", + "the free offer uses BillOverUsage or AutoPause, never BillForUsage", + "a bare HTTP 500 from a create carrying --use-free-limit is read as no free slot rather than as an outage", + "the connection string names the user database, not master, and no password is handed back in it", + "a production answer uses a Microsoft Entra managed identity rather than a SQL login with a password in the connection string", + "success is reported only after SERVERPROPERTY('EngineEdition') has returned 5 over the connection string being handed back", + "Msg 40613 on the first connect is read as a serverless resume and retried, not as a provisioning failure", + "if any identity other than the administrator must connect, the plan names the principal it gets, a contained database user or an Entra login in master, and who has to run it", + "Hyperscale is handed to provision-hyperscale rather than answered here" + ], + "probes": [ + { + "cmd": "az sql db create --help 2>&1 | grep -qi 'firewall' && exit 1 || exit 0", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The claim the whole skill turns on, made executable and made safe: az sql db create has no firewall argument at all, so a four-step plan is not padding. Measured on Azure CLI 2.90.0 on 2026-09-03, help output only, no request reaches Azure and nothing is provisioned. Watched going red the same day by pointing the same grep at az sql server firewall-rule create --help, which does match." + }, + { + "cmd": "az sql db create --help 2>&1 | tr -s ' \\n' ' ' | grep -q 'Allowed on one database in a subscription'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The CLI's own help for --use-free-limit says the free offer is allowed on ONE database in a subscription, while the free offer documentation says 10 in three separate places. This probe pins the wrong statement so the skill can be trusted when it contradicts the tool. If a future CLI fixes the help, this probe goes red and the skill's paragraph should be deleted rather than the probe. tr collapses the help's wrapped lines before matching. Measured on Azure CLI 2.90.0, 2026-09-03." + }, + { + "cmd": "az sql db create -g rg-nonexistent-probe -s srv-nonexistent-probe -n probedb -e GeneralPurpose -f Gen5 -c 2 --compute-model Serverless --use-free-limit --free-limit-exhaustion-behavior BillForUsage 2>&1 | grep -q 'is not a valid value'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "BillForUsage, the value the CLI's own help text describes, is rejected by argument parsing before authentication or any network call: 'BillForUsage' is not a valid value for '--free-limit-exhaustion-behavior'. Allowed values: AutoPause, BillOverUsage. Measured on Azure CLI 2.90.0, 2026-09-03. The resource group and server names are deliberately nonexistent so that if a future CLI ever accepts the value, the probe fails on ResourceGroupNotFound and still creates nothing." + }, + { + "cmd": "az sql db show-connection-string -c ado.net -s probesrv -n probedb -o tsv | grep -q 'Initial Catalog=probedb;.*Encrypt=true'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "show-connection-string is a local template generator, not an ARM call, so this asserts the two things the skill tells a reader to check in its output without touching a subscription: Initial Catalog carries the user database rather than master, and Encrypt=true survives. Measured on Azure CLI 2.90.0, 2026-09-03. Watched going red by asking for -n master, where the Initial Catalog half stops matching." + }, + { + "cmd": "az sql db show-connection-string --help 2>&1 | tr -s ' \\n' ' ' | grep -q 'Allowed values: ADIntegrated, ADPassword, SqlPassword'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "Why the skill tells a reader to edit the connection string by hand for production. The --auth-type flag offers exactly three modes and none of them is a managed identity, so the CLI cannot emit Authentication=\"Active Directory Managed Identity\", which Microsoft Learn calls the recommended method for programmatic access to SQL. Measured on Azure CLI 2.90.0, 2026-09-03. Watched going red by appending a fourth value to the expected list." + }, + { + "sql": "SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS engine_edition;", + "expect": { + "rows": [ + [ + 5 + ] + ] + }, + "why": "The reachability check the skill's 'Check it worked' section hands the reader, run as a probe so the number in the skill is the number the engine returns. Microsoft Learn documents EngineEdition 5 as SQL Database, and 5 is what the Azure SQL Database container reports too, which is why this probe runs on both targets rather than cloud only. A reader who gets 5 back over their new connection string has proved reachability and product in one round trip." + } + ] + }, + "triggering": { + "explicit": ["Use the provision-azure-sql-db skill"], + "implicit": [ + "create an Azure SQL database for me", + "set up a free Azure SQL database", + "provision a SQL database in Azure and give me the connection string" + ], + "routing": [ + "I need a database in Azure for this app", + "spin up SQL in my subscription" + ], + "negative": [ + "run Azure SQL locally in a container", + "create a PostgreSQL database in Azure", + "resize my existing database to Hyperscale" + ] + } +} diff --git a/plugins/microsoft-sql-fdh/skills/provision-hyperscale b/plugins/microsoft-sql-fdh/skills/provision-hyperscale deleted file mode 120000 index 69975b9..0000000 --- a/plugins/microsoft-sql-fdh/skills/provision-hyperscale +++ /dev/null @@ -1 +0,0 @@ -../../../skills/provision-hyperscale \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/provision-hyperscale/SKILL.md b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/SKILL.md new file mode 100644 index 0000000..a1ba9d0 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/SKILL.md @@ -0,0 +1,327 @@ +--- +name: provision-hyperscale +description: >- + Puts a real workload on the Hyperscale service tier of Azure SQL Database: when to choose it, how + to size it, and how to convert an existing database without walking through a door that does not + open again. Use when a user asks "should I use Hyperscale", "convert my database to Hyperscale", + "we are close to the 4 TB ceiling", "how many vCores for Hyperscale", "Hyperscale serverless or + provisioned", "add a read replica", "elastic pool for Hyperscale", or "can I go back from + Hyperscale". Also use when a free offer database has to become a real one, because the free limit + must be turned off before the tier can change and that step cannot be undone. Covers the + eligibility rules for reverse migration, controlling the cutover, replica-based read scale-out, + and what actually limits write throughput. Creating the server, the database and the firewall + rule belongs to provision-azure-sql-db. +--- + +# Put a workload on Hyperscale + +Choosing Hyperscale is easy. Two things go wrong: **the way you arrive decides whether you can ever +leave**, and **scaling up does not buy the throughput people expect**. + +Verified on 2026-08-28 against a live Azure SQL Database logical server with Azure CLI 2.89.1, and +every flag below was re-read from Azure CLI 2.90.0 help on 2026-09-03. + +Hyperscale exists in Azure SQL Database, and not in Azure SQL Managed Instance. + +## What this skill does not own + +| Question | Where | +|---|---| +| Creating the logical server, the database, the firewall rule, the free offer, and how many free databases a subscription gets | `provision-azure-sql-db` | +| Retry, pool sizing, connection strings, what `ApplicationIntent` means per driver | `connect-to-azure-sql` | +| A database that is already slow or already throttled | `diagnose-resource-pressure` | +| Running a restore, and why `BACKUP` and `RESTORE` T-SQL are refused on the service | `restore-and-recover` | + +## The decision: is this a Hyperscale workload + +The service tier comparison names Hyperscale **the recommended and default service tier for all new +and modernizing OLTP and HTAP workloads**, reversing the older advice to start on General Purpose. +That does not make it automatic. Four things decide it: + +| Signal | Reading | +|---|---| +| Data over 4 TB, or growth you cannot forecast | Hyperscale. General Purpose and Business Critical stop at 4 TB; Hyperscale runs 10 GB to 128 TB | +| Backup or restore time is the problem | Hyperscale. Backups are storage snapshots, so an in-region restore takes minutes at any size. A **geo**-restore is still size-of-data | +| Read-heavy reporting beside the write workload | Hyperscale, with replicas. Up to 4 high availability replicas and up to 30 named replicas | +| Small, stable, cost-sensitive, may need to move back | Not yet. Read the next section before creating anything | + +Three counter-signals that surprise people: + +- **Durable and non-durable memory-optimized tables are not supported.** A database holding any + In-Memory OLTP objects cannot be converted from Premium or Business Critical until they are + dropped. Memory-optimized table types, table variables and natively compiled modules do come. +- **`DBCC CHECKDB` is not available.** `DBCC CHECKTABLE ('TableName') WITH TABLOCK` is the + documented workaround. +- **Restore does not cross the tier boundary either way**: no restore into Hyperscale, and none out + of it. + +## The one-way door, which is about how you arrived + +**This is the correction.** "Hyperscale is one way" is not true, and neither is "it is a tier change +like any other". The exit depends on how the database got there, and both cases were run. + +| How the database got to Hyperscale | Can it leave | +|---|---| +| **Converted** from another tier | Yes, for 45 days after the conversion, to General Purpose only, single database only, 2 vCores or more, no geo-replication and no named replicas | +| **Created** as Hyperscale | **No.** Refused outright, at any age | + +A converted database comes back with a plain update, which ran successfully in about nine minutes +while empty: + +```bash +az sql db update -g -s -n \ + --edition GeneralPurpose --service-objective GP_Gen5_2 --compute-model Provisioned +``` + +The same command against a database **created** as Hyperscale is refused in about two seconds: + +``` +ERROR: (ProvisioningDisabled) Hyperscale database '' is ineligible for reverse +migration. Database created as brand-new Hyperscale application. +``` + +**What being wrong costs.** The only route out of an ineligible database is a bacpac export and +import, which the portal, PowerShell, the Azure CLI and the REST API **do not support** for +Hyperscale, leaving a client-side tool documented to about 150 GB. If the workload might not stay, +create General Purpose and convert. + +## The free offer funnel step + +**The free limit must be turned off before the tier can change.** The free offer runs on General +Purpose serverless, and setting the free limit on a Hyperscale database is refused: + +``` +ERROR: (ProvisioningDisabled) Provisioning of free limit database is not supported for provided +service level objective or region +``` + +Turn the free offer off first with `--use-free-limit false`, then change the tier. + +**Turning it off is permanent.** Microsoft's wording is exact: "Once you convert a free offer +database to a paid service tier, you can't revert to the free offer." Converting an existing +database *into* the free offer is unsupported too, so there is no way back. A developer who converts +a free database straight into a **newly created** Hyperscale database has closed two doors in one +afternoon. Say so before running the command. + +**Two sources disagree on the cap, so quote neither.** The free offer FAQ says "There's a limit of +10 free databases per subscription"; `az sql db create --help` says `--use-free-limit` is "Allowed +on one database in a subscription". `provision-azure-sql-db` owns that count. + +## Sizing + +Service objectives are `HS_Gen5_` for standard-series provisioned, `HS_S_Gen5_` for +serverless, `HS_PRMS_` for premium-series and `HS_MOPRMS_` for premium-series memory +optimized. Ask the service rather than recalling the list: + +```bash +az sql db list-editions -l -e Hyperscale \ + --query "[].supportedServiceLevelObjectives[].{slo:name,family:sku.family}" -o table +``` + +### Gotcha: the hardware family names are not the ones the help text offers + +`az sql db create --help` says `--family` accepts `Gen4, Gen5`. For Hyperscale the real families +include **`8IM`** (premium-series) and **`8IH`** (premium-series memory optimized). `--family PRMS` +is rejected client side, and the rejection prints the true list: + +``` +ERROR: Could not find sku in tier 'Hyperscale' with family 'PRMS', capacity 2. +Supported families & capacities for 'Hyperscale' are: [('Gen5', 2), ('8IM', 2), ('8IH', 2), ...] +``` + +Prefer `--service-objective HS_PRMS_8`, which does not depend on the spelling at all. + +### Gotcha: more vCores does not buy more write throughput + +The **maximum log rate is flat across every compute size** in a series. The resource limits table +reads 100 MiB/s for standard-series at 2 vCores and 100 MiB/s again at 80. Premium-series is +150 MiB/s at every size. So scaling 8 vCores to 32 changes CPU, memory, workers and local IOPS, and +changes the write ceiling **not at all**. What helps is premium-series hardware, generating less log +(batching, fewer indexes on the write path), or splitting the workload. + +What does scale with vCores, on standard-series: + +| Per compute size | Value | +|---|---| +| Max concurrent workers | 100 per vCore provisioned, 75 per max vCore serverless | +| Max concurrent sessions | 30,000 at every size, so never the binding limit | +| tempdb, memory, local SSD IOPS | Proportional to vCores | +| Max data size | 128 TB at every size | + +Pool sizing against workers rather than sessions belongs to `connect-to-azure-sql`. + +### Storage: there is no size to choose + +A Hyperscale database is created with **no max size**, and `maxSizeBytes` reads `-1` after both +conversion and creation. It starts at 10 GB, grows in 10 GB steps to 128 TB, and bills for what is +allocated. The Hyperscale FAQ answers **no** to setting a hard cap on data growth, so `--max-size` +is a General Purpose habit that gives a plan a limit the tier lacks. + +### Serverless or provisioned + +Serverless Hyperscale is **standard-series only**; premium-series has no serverless compute tier. +Serverless suits intermittent and development workloads, provisioned steady ones. The compute tier +can change later; the exit rules above do not. + +## Read scale-out is a replica count, not a switch + +**The second silent failure, verified by running it.** Read scale-out follows the high availability +replica count: + +```bash +# provision the replica that read-intent connections will land on +az sql db update -g -s -n --ha-replicas 1 +``` + +1. `--ha-replicas 0` flips `readScale` to `Disabled` on its own; back to 1 flips it to `Enabled`. + The replica count is the control. +2. **`--read-scale Disabled` on a Hyperscale database is accepted and does nothing.** It returns + success and the property stays `Enabled`. The flag's help says it "is only settable for Premium + and Business Critical databases", and on Hyperscale it fails quietly instead of erroring. +3. With zero replicas an `ApplicationIntent=ReadOnly` connection is **not** refused. Microsoft's + wording: it "is routed to the primary replica and defaults to the `ReadWrite` behavior". The read + offload silently does not happen and the only symptom is a busier primary. + +A default conversion or creation gives **1** high availability replica, so read scale-out is on +unless someone turned it off. + +### Named replicas, when reporting needs its own compute + +Up to 30 per primary, each a separate read-only database with its own service objective and logins. +No data copy, so one appears in about a minute, and scaling either side does not disconnect the +other's users: + +```bash +az sql db replica create -g -s -n --secondary-type Named \ + --partner-server --partner-database _reporting \ + --service-objective HS_Gen5_2 +``` + +The same from `master`, with no Azure credentials at hand: + +```sql +ALTER DATABASE [] + ADD SECONDARY ON SERVER [] + WITH (SERVICE_OBJECTIVE = 'HS_Gen5_2', SECONDARY_TYPE = Named, + DATABASE_NAME = [_reporting]); +``` + +A named replica has no high availability replica unless you add `--ha-replicas 1`, cannot join an +elastic pool, and blocks reverse migration while it exists. + +## Converting an existing database + +Two phases: a data copy while the source stays online, then a cutover. Take control of the cutover. + +```bash +# 1. start the conversion, and keep the cutover in your hands +az sql db update -g -s -n \ + --edition Hyperscale --service-objective HS_Gen5_2 --manual-cutover + +# 2. watch it. This is the only place the real state shows up +az sql db op list -g -s -d -o table + +# 3. cut over, within 24 hours of the database being ready +az sql db update -g -s -n --perform-cutover +``` + +### Gotcha: the database reports the tier it is going to, not the one it is on + +During the conversion `az sql db show` reports `sku` and `edition` as the **target**, while +`currentSku` and `currentServiceObjectiveName` still report what is running and billing: + +```json +{ "edition": "Hyperscale", "requestedServiceObjectiveName": "HS_Gen5_2", + "currentServiceObjectiveName": "GP_S_Gen5_2", "status": "Online" } +``` + +An agent that queries `sku` has reported success before the cutover happened. The pending manual +cutover is not in the database resource at all: `manualCutover` reads `null` throughout. + +- **Billing begins only after cutover**, so waiting is not expensive, but the source switches from + asynchronous to synchronous copying while it waits, which can raise write latency under load. +- The manual cutover window is **24 hours** from the moment the database is ready. +- Final cutover is usually **under a minute** of downtime; converting from Premium or Business + Critical drops connections during the first phase, which retry logic covers. + +### Before you start + +- **Basic tier cannot convert directly.** Move to another tier first. +- **Geo-replication**: start from the primary. Converting a geo-secondary is refused, and the + secondary converts with the primary. Reduce to one geo-secondary and remove replica chains. +- **Elastic pools**: a zone-redundant pool cannot move into one that is not. Leave the pool first. +- **Compatibility level does not change.** + +## Elastic pools, briefly + +Right for many small databases with uneven demand. Four rules that catch people: + +- Adding a non-Hyperscale database to a Hyperscale pool **converts** it. +- A pool of another tier cannot become a Hyperscale pool, or the reverse. Create the pool you want. +- **Zone redundancy is settable only at creation**, for the pool and for the database. +- A pooled database must leave the pool before reverse migration, and the pool's log rate is one + 125 MiB/s ceiling shared by everything in it. + +## Check it worked + +A command that returned is not a database on Hyperscale. + +**The tier the engine itself reports.** Connect to the database and ask the catalog view: + +```bash +sqlcmd -S .database.windows.net -d -U -P "$SQLCMD_PASSWORD" -Q \ + "SELECT d.name, dso.edition, dso.service_objective + FROM sys.database_service_objectives AS dso + JOIN sys.databases AS d ON dso.database_id = d.database_id + WHERE d.name = DB_NAME();" +``` + +Expect exactly one row, `edition` reading `Hyperscale` and `service_objective` reading the objective +you asked for, `HS_Gen5_2` above. Any other edition means the cutover has not happened. From +`master` the same query returns every database on the server. Microsoft Learn says the view returns +data only on Azure SQL Database, so this is not a check to run against the local container. + +**The cutover, if you asked for a manual one.** From `master`: + +```bash +sqlcmd -S .database.windows.net -d master -U -P "$SQLCMD_PASSWORD" -Q \ + "SELECT TOP 5 operation, state_desc, phase_desc, percent_complete + FROM sys.dm_operation_status + WHERE major_resource_id = '' ORDER BY start_time DESC;" +``` + +While the conversion waits, `phase_desc` reads `WaitingForCutover`; after `--perform-cutover` the +newest row's `state_desc` reads `Completed`. No rows means no recent operation, which is not a +finished one. + +**Storage and replicas, which only the control plane knows:** + +```bash +az sql db show -g -s -n --query \ + "{cur:currentServiceObjectiveName, req:requestedServiceObjectiveName, max:maxSizeBytes, ha:highAvailabilityReplicaCount, rs:readScale}" -o table +``` + +Expect `cur` and `req` to match, `max` to read `-1`, and `ha` to be at least `1` if anything +connects with `ApplicationIntent=ReadOnly`. `cur` different from `req` is a conversion still in +flight, whatever `edition` says. + +## Do not + +- Do not create a new database directly in Hyperscale for a workload that might have to move back. +- Do not call the conversion one way, or call it reversible. State which case this database is in. +- Do not turn off a free offer without saying so first. It cannot be turned back on. +- Do not use `--read-scale` on Hyperscale, and do not promise read offload at 0 replicas. Both are + accepted, ignored, and reported by nothing. +- Do not scale vCores to fix a write ceiling before checking the log rate, and do not pass a max + size. +- Do not report a conversion finished on the strength of the `sku` field. + +## References + +- Before you tell anyone a Hyperscale database can or cannot go back, open + [references/exit-paths-and-eligibility.md](references/exit-paths-and-eligibility.md) for the full + eligibility matrix, what repeated conversions do to restorable backups, the verified transcripts, + and the pinned Microsoft Learn sources. + +Resource limits, premium-series regional availability and the preview boundaries move. Fetch the +resource limits article rather than recalling numbers from this file. diff --git a/plugins/microsoft-sql-fdh/skills/provision-hyperscale/references/exit-paths-and-eligibility.md b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/references/exit-paths-and-eligibility.md new file mode 100644 index 0000000..29ac35c --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/references/exit-paths-and-eligibility.md @@ -0,0 +1,137 @@ +# Leaving Hyperscale: eligibility, backups, and what was actually run + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [The eligibility matrix](#the-eligibility-matrix) +- [What the service says when it refuses](#what-the-service-says-when-it-refuses) +- [What happens to backups across repeated moves](#what-happens-to-backups-across-repeated-moves) +- [The bacpac fallback, and its real limits](#the-bacpac-fallback-and-its-real-limits) +- [Verified transcript](#verified-transcript) +- [Sources](#sources) + +## How to use this file + +Read it before answering "can we go back". The answer is not a property of Hyperscale. It is a +property of the individual database, and the deciding fact is how that database arrived. + +## The eligibility matrix + +Every row has to be true at once. One false row means reverse migration is refused, and the refusal +happens before anything starts. + +| Condition | Required for reverse migration | +|---|---| +| Origin | The database was **converted** from another service tier. One created as Hyperscale is never eligible | +| Age | Within **45 days** of the original conversion to Hyperscale | +| Target tier | **General Purpose only**. Provisioned or serverless compute, either is allowed | +| Target size | **2 vCores or more**. Scale below that only after the move completes | +| Elastic pool | Not in a pool, on either side. Remove the database from the Hyperscale pool first | +| Geo-replication | Not enabled | +| Named replicas | None | +| Allocated size | Small enough to fit the target service objective, and inside `--max-size` if one is given | + +To reach Business Critical or a DTU tier, reverse migrate to General Purpose first, then change tier +again. + +Duration is a size-of-data operation, not a metadata switch. It depends on database size, on write +activity during the move, and on the vCores given to the target, which should be at least the vCores +the Hyperscale database had. The source can be throttled on transaction log rate while the move +progresses. Final cutover is a few minutes of downtime. + +## What the service says when it refuses + +Verbatim, from a live attempt on 2026-08-28: + +``` +ERROR: (ProvisioningDisabled) Hyperscale database '' is ineligible for reverse migration. +Database created as brand-new Hyperscale application. +``` + +The database name is the only thing elided. The message names the reason, which is worth reading +rather than retrying. "Created as brand-new Hyperscale application" is the origin rule, and no wait, +flag or support request changes it. + +The free offer refusal, from the same session, is a different message with a similar shape: + +``` +ERROR: (ProvisioningDisabled) Provisioning of free limit database is not supported for provided +service level objective or region +``` + +That one is about the combination, not about a permission. The free offer runs on General Purpose +serverless, so the free limit and Hyperscale cannot both be set. + +## What happens to backups across repeated moves + +This is the part that gets discovered late. Only the backups of the **current** and the +**immediately previous** tier of a database are available for restore. + +Microsoft's own worked example: General Purpose, convert to Hyperscale, reverse migrate to General +Purpose, change to Business Critical, convert to Hyperscale, reverse migrate to General Purpose. At +the end, only the backups from the last two steps exist. Everything earlier is gone, and it is gone +**as soon as a reverse migration starts**, and stays gone even if that migration is cancelled. + +Two consequences: + +- Repeated experimentation between tiers quietly destroys restore points. Do the experiment on a + copy. +- Pre-migration backups taken before the move to Hyperscale live for the source database's retention + period and can be restored to any tier that is not Hyperscale, from the command line. That is a + real, time-limited safety net, and it is separate from the 45-day reverse migration window. + +Also note the tier boundary in restore: a database that is not Hyperscale cannot be restored **as** +Hyperscale, and a Hyperscale database cannot be restored as anything else. + +## The bacpac fallback, and its real limits + +For a database that does not qualify, the documented route out is export and import, and the +documentation is unusually blunt about the gaps: + +- bacpac export and import for Hyperscale is **not supported** from the portal, from PowerShell's + export and import cmdlets, from the Azure CLI export and import commands, or from the REST API. +- It **is** supported from a client-side tool and the SqlPackage command line, version 18.4 or + later, for databases **up to about 150 GB**. +- Above that, the documentation says the operation might take a long time and can fail for various + reasons. + +Other data movement paths exist, bulk copy and pipeline tooling among them, and they are the +realistic answer for a large database. None of them is a tier change, and all of them mean a new +database and a cutover of their own. Plan for that rather than discovering it. + +## Verified transcript + +Run on 2026-08-28 with Azure CLI 2.89.1 against a throwaway logical server, then deleted. Summarised +rather than pasted, because the values were environment specific. + +| Step | Command | Result | +|---|---|---| +| Create General Purpose serverless | `az sql db create ... --edition GeneralPurpose --compute-model Serverless --family Gen5 --capacity 2` | Online, `GP_S_Gen5_2`, max size 32 GB | +| Convert with manual cutover | `az sql db update ... --edition Hyperscale --service-objective HS_Gen5_2 --manual-cutover` | Operation `InProgress` at 50 percent, `currentServiceObjectiveName` still `GP_S_Gen5_2`, `sku` already `HS_Gen5` | +| Cut over | `az sql db update ... --perform-cutover` | `HS_Gen5_2`, `highAvailabilityReplicaCount` 1, `readScale` Enabled, `maxSizeBytes` -1 | +| Try the free limit on it | `az sql db update ... --use-free-limit true` | Refused, `ProvisioningDisabled` | +| Try to disable read scale-out | `az sql db update ... --read-scale Disabled` | Accepted, and `readScale` stayed `Enabled` | +| Remove the replica | `az sql db update ... --ha-replicas 0` | `readScale` became `Disabled` on its own | +| Reverse migrate | `az sql db update ... --edition GeneralPurpose --service-objective GP_Gen5_2 --compute-model Provisioned` | Succeeded in about nine minutes, max size back to 1 TB | +| Create one directly as Hyperscale | `az sql db create ... --edition Hyperscale --family Gen5 --capacity 2` | Online in about six minutes, replica count 1, max size -1 | +| Reverse migrate that one | same update command | Refused in about two seconds | +| Premium-series family | `--family PRMS` rejected client side, `--family 8IM` created `HS_PRMS_2` | The help text's `Gen4, Gen5` is not the whole list | + +## Sources + +Fetch these rather than trusting the summary above, which carries a verification date for a reason. + +- Convert a database to Hyperscale: + +- Reverse migrate from Hyperscale, including the eligibility list and the backup rules: + +- Free offer FAQ, for the tier-change step and the fact it cannot be undone: + +- Hyperscale service tier, for capabilities, replica counts and current limitations: + +- Hyperscale secondary replicas, for read scale-out behaviour and named replicas: + +- Single database vCore resource limits, for log rate, workers and tempdb per compute size: + +- Hyperscale elastic pools: + diff --git a/plugins/microsoft-sql-fdh/skills/provision-hyperscale/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/skill.spec.jsonc new file mode 100644 index 0000000..5bee33e --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/provision-hyperscale/skill.spec.jsonc @@ -0,0 +1,130 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "provision-hyperscale", + "domain": "provisioning-and-local-dev", + "value": ["not-in-training-data", "corrects-a-wrong-assumption"], + "correction": "Agents treat Hyperscale as an ordinary service objective change, reversible like any other, and treat the free offer as a starting point that grows into it. Neither holds. A free offer database cannot change tier at all until the free limit is turned off, and turning it off is permanent: the database can never return to the free offer. Coming back out of Hyperscale is not a tier change either. Reverse migration exists only for a database that was CONVERTED to Hyperscale, only within 45 days of that conversion, only to General Purpose, only outside an elastic pool and only at 2 vCores or more; a database created as Hyperscale is refused outright with 'ineligible for reverse migration. Database created as brand-new Hyperscale application.' The cost is a one-way door walked through by accident, because the exit from a brand-new Hyperscale database is a bacpac export and import that the portal, the CLI, PowerShell and the REST API do not support for Hyperscale at all, leaving a client-side tool documented as reliable only to about 150 GB.", + "value_probe": { + "gate": "The answer names which moves in this plan cannot be undone before recommending any of them, and treats getting back to a smaller tier later as a separate question with conditions of its own rather than as another size change.", + "task": "We prototyped on the free monthly Azure SQL Database and it has outgrown that: we are a few hundred gigabytes now and still growing. Our architect wants us on the tier that scales storage out for databases this size, and finance has asked whether we can move this month and then move back down next quarter if the bill turns out worse than they expect. Tell me what I would actually have to do, in order, and flag anything I would not be able to undo, because I have to take this to them on Thursday.", + "states": [ + "Turning off the free limit is permanent and the database can never return to the free offer afterwards.", + "A database on the free offer cannot change service tier at all while the free limit is still in place.", + "Any path back to a smaller tier is time limited and carries conditions on the target tier, the pool and the vCore count.", + "Exporting and importing the database as a portable file is the fallback exit, and it has tooling limits of its own." + ], + "refutes": [ + "A database can be moved back down from that tier at any time with an ordinary service objective change.", + "The free offer is a starting point that simply grows into a larger tier when the workload needs it.", + "A database created directly on that tier has the same path back as one that was converted to it.", + "The portal, the CLI, PowerShell or the REST API can export this database to a portable file." + ] + }, + "posture": ["read", "write", "provision"], + "maturity": "ga", + "applies_to": ["azure-sql-db"], + "related": [ + "provision-azure-sql-db", + "connect-to-azure-sql", + "hyperscale-target-sizing", + "diagnose-resource-pressure", + "restore-and-recover" + ], + "validation": { + "target": "both", + "setup": [ + "a General Purpose serverless database on a logical server the caller can update", + "a second database created directly in the Hyperscale service tier" + ], + "assert": [ + "setting the free limit on a Hyperscale database is refused, so the free offer must be turned off before the tier changes", + "a converted database returns to General Purpose with a plain edition and service objective update, so the conversion is not one way for every database", + "a database created as Hyperscale is refused reverse migration, naming it as created brand new", + "read scale-out follows the high availability replica count, and the read-scale switch is accepted and ignored on Hyperscale", + "a Hyperscale database reports no maximum size, so no max size is chosen at creation", + "the premium-series hardware families are not the values the create command's help text lists" + ], + "probes": [ + { + "where": "host", + "cmd": "az sql db create --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'Allowed values include: Gen4, Gen5'", + "expect": { "succeeds": true }, + "why": "The sizing gotcha only makes sense while the help text really does offer Gen4 and Gen5 and nothing else for --family, which is why the body tells a reader to pass --service-objective instead. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0. If Azure CLI ever lists 8IM and 8IH here the gotcha is stale and this probe goes red first. The Hyperscale-only families themselves cannot be probed without creating a database, which this skill is not allowed to do." + }, + { + "where": "host", + "cmd": "az sql db create --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'Allowed on one database in a subscription'", + "expect": { "succeeds": true }, + "why": "The free offer section quotes this line to show that the Azure CLI help and the free offer FAQ disagree about how many free databases a subscription gets, which is why the body quotes neither as the answer. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0." + }, + { + "where": "host", + "cmd": "az sql db update --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'only settable for Premium and Business Critical databases'", + "expect": { "succeeds": true }, + "why": "The body quotes this sentence as the reason --read-scale is accepted and ignored on Hyperscale. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0. The silent no-op itself needs a live Hyperscale database to observe, so this probe pins the quoted wording rather than the behaviour." + }, + { + "where": "host", + "cmd": "az sql db update --help 2>&1 | grep -q -- '--ha-replicas'", + "expect": { "succeeds": true }, + "why": "Read scale-out in this skill is configured entirely through --ha-replicas, so the flag existing is the precondition for that whole section. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0." + }, + { + "where": "host", + "cmd": "az sql db update --help 2>&1 | grep -q -- '--manual-cutover' && az sql db update --help 2>&1 | grep -q -- '--perform-cutover'", + "expect": { "succeeds": true }, + "why": "The conversion procedure is three commands and two of them depend on these flags, which are what keep the cutover in the caller's hands instead of the service's. Measured 2026-09-03 on Azure CLI 2.90.0, both present, exit 0." + }, + { + "where": "host", + "cmd": "az sql db replica create --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'Allowed values include: Geo, Named'", + "expect": { "succeeds": true }, + "why": "The named replica command in the body passes --secondary-type Named, and Named is a Hyperscale-only value that a reader coming from geo-replication will not expect to find here. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0." + }, + { + "sql": "SELECT COUNT(*) AS n, MAX(CASE WHEN dso.edition = 'Hyperscale' THEN 1 ELSE 0 END) AS is_hyperscale FROM sys.database_service_objectives AS dso JOIN sys.databases AS d ON dso.database_id = d.database_id WHERE d.name = DB_NAME();", + "expect": { "rows": [[1, 0]] }, + "why": "The Check it worked section hands a reader this join and tells them to expect exactly one row whose edition reads Hyperscale. The join, the two column names and the one-row shape are settled here; the Hyperscale value is not, and the second column is what says so, because a container database reports GeneralPurpose and GP_Gen5_2. Measured 2026-09-05 on 18.0.226_4_147: one row, edition GeneralPurpose, service_objective GP_Gen5_2. Red if sys.database_service_objectives stops resolving, if edition or service_objective is renamed, if the filter on DB_NAME() stops returning exactly one row, or if a container ever reported itself as Hyperscale, which would make the skill's warning that this is not a container check the wrong warning." + }, + { + "sql": "SELECT TOP 1 operation, state_desc FROM sys.dm_operation_status WHERE major_resource_id = DB_NAME() ORDER BY start_time DESC;", + "expect": { "rows": [["CREATE DATABASE", "COMPLETED"]] }, + "why": "The cutover check in Check it worked reads sys.dm_operation_status filtered on major_resource_id and ordered by start_time, and says a state_desc of Completed is what a finished operation looks like. Those four names and that shape are settled here against the one operation a scratch database is guaranteed to have, its own CREATE DATABASE. Measured 2026-09-05: CREATE DATABASE, COMPLETED. Red if the view is gone, if operation, state_desc, major_resource_id or start_time is renamed, or if the engine stops recording the create." + }, + { + "sql": "SELECT TOP 1 phase_desc FROM sys.dm_operation_status;", + "expect": { "error": 207 }, + "why": "The same check tells a reader phase_desc reads WaitingForCutover while a manual cutover waits. Microsoft Learn documents phase_desc and phase_code on sys.dm_operation_status as Hyperscale conversion columns, and this build does not carry the column at all: measured 2026-09-05, Msg 207, Invalid column name 'phase_desc'. So the cutover half of that check is settled only by a real Azure SQL Database and this probe records the boundary rather than implying it was tested. Red if a later build adds the column, which is the day the claim becomes locally checkable." + }, + { + "sql": "ALTER DATABASE [probe_provision_hyperscale] ADD SECONDARY ON SERVER [probe_partner_server] WITH (SERVICE_OBJECTIVE = 'HS_Gen5_2', SECONDARY_TYPE = Named, DATABASE_NAME = [probe_provision_hyperscale_reporting]);", + "expect": { "error": 41920 }, + "why": "The named replica section prints this statement for a reader with no Azure credentials at hand, so at minimum the engine has to accept it as T-SQL. It does: measured 2026-09-05 the statement is refused by feature, Msg 41920, while the same statement with SECONDARY_TYPE misspelled is refused by the parser, Msg 102. A feature refusal rather than a syntax error is the whole evidence available locally, because a named replica needs a logical server. Red if this build ever answers Msg 102, which would mean the clause order or the option names in the skill are wrong." + } + ] + }, + "triggering": { + "explicit": ["Use the provision-hyperscale skill"], + "implicit": [ + "should I put this database on Hyperscale", + "convert my Azure SQL database to Hyperscale", + "how many vCores should I pick for Hyperscale", + "can I go back after moving to Hyperscale", + "my free Azure SQL database has outgrown the free offer and needs a bigger tier", + "add a read replica to my Hyperscale database" + ], + "routing": [ + "our database is about to hit the four terabyte ceiling", + "writes are the bottleneck and scaling up did not help", + "we want one pool for a lot of tenant databases that each grow unpredictably" + ], + "negative": [ + "create an Azure SQL database and open the firewall for my IP", + "give me a connection string for my new database", + "this query got slow after the last deployment", + "add retry logic to my data access layer", + "run Azure SQL locally in a container", + "restore my database to how it looked yesterday morning" + ] + } +} diff --git a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely deleted file mode 120000 index fb347b2..0000000 --- a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely +++ /dev/null @@ -1 +0,0 @@ -../../../skills/schema-migrations-safely \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/SKILL.md b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/SKILL.md new file mode 100644 index 0000000..4a2acb3 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/SKILL.md @@ -0,0 +1,348 @@ +--- +name: schema-migrations-safely +description: >- + Decides whether a schema change is safe to apply to a live Azure SQL Database, and rewrites the + migration so it is. Use when someone asks "is this migration safe to run in production", "can I + add this column without downtime", "zero downtime schema change", "expand and contract", "blue + green database deploy", "make this migration re-runnable", "should the app run migrations at + startup", or "how do I roll this back"; and when a deployment blocked every query, instances + fought over the same migration, or a retry applied its backfill twice. Covers metadata-only + versus table-rewriting alterations, the schema lock that blocks readers under snapshot + isolation, ONLINE and RESUMABLE, and why a guard is not a guard under concurrency or retry. + Tooling is sql-database-projects and github-actions-for-sql, ORM migrations their own skills, + the table design design-azure-sql-schema. +--- + +# Apply a schema change to a live Azure SQL Database + +The doctrine every migration tool inherits and none enforces, whether the change travels by project +publish, pipeline step, ORM migration or hand written script. + +Timings and log bytes measured 2026-08-28 on a live engine reporting +`SERVERPROPERTY('EngineEdition')` 5 and Edition `SQL Azure`, compatibility level 170, read committed +snapshot isolation on, over a 2,000,000 row 230 MB table. Options checked against +Microsoft Learn the same week; every statement below runs on Azure SQL Database. + +## The correction + +An agent asked whether a migration is safe answers with expand and contract, adds `IF NOT EXISTS` +guards, and calls the script idempotent. All three break here, silently: + +1. **`ALTER TABLE` is not uniformly cheap, and the expensive ones do not look expensive.** On 2 + million rows a `NOT NULL` column with a constant default took 1.7 ms; widening `varchar(50)` to + `varchar(max)` took 7.3 s and wrote 572 MB of log. +2. **The lock a schema change takes is the one thing snapshot isolation does not save you from.** + A plain, a `SNAPSHOT` and a `NOLOCK` read all waited on it. +3. **A guard protects the statement it wraps and nothing else.** Eight instances starting at once + all passed the same `IF NOT EXISTS` checks and applied the same backfill eight times. + +## 1. Which alterations are metadata only, and which rewrite every row + +Log bytes are the honest signal: metadata only writes kilobytes whatever the row count; a rewrite +writes more log than the table holds. + +| Change | Time | Log written | +|---|---|---| +| `ADD` nullable column, no default | 3.6 ms | 1.2 KB | +| `ADD` `NOT NULL` column, **constant** default | 1.7 ms | 2.2 KB | +| `ADD` `NOT NULL` column, `SYSUTCDATETIME()` default | 1.5 ms | 2.2 KB | +| `ALTER COLUMN` widen `varchar(20)` to `varchar(50)` | 1.0 ms | 1.1 KB | +| `ADD CONSTRAINT ... CHECK` `WITH NOCHECK` | 1.5 ms | 1.4 KB | +| `DROP COLUMN` | 2.9 ms | 1.2 KB | +| `ADD CONSTRAINT ... CHECK`, validated | 521 ms | 1.7 KB (scan, no writes) | +| `CREATE INDEX`, offline | 628 ms | 37 MB | +| **`ADD` `NOT NULL` column, `NEWID()` default** | **4.6 s** | **448 MB** | +| **`ALTER COLUMN` `int` to `bigint`** | **6.1 s** | **509 MB** | +| **`ALTER COLUMN` `varchar` to `nvarchar`** | **5.1 s** | **426 MB** | +| **`ALTER COLUMN` `varchar(50)` to `varchar(max)`** | **7.3 s** | **572 MB** | +| **`ALTER COLUMN` `NULL` to `NOT NULL`** | **14.5 s** | **661 MB** | + +The line is not "adding is cheap, changing is expensive". Learn's rule, which the measurements obey: +a `NOT NULL` column with a default is metadata only when the default is a **runtime constant**, the +same value for every row. `SYSUTCDATETIME()` is one; `NEWID()` and `NEWSEQUENTIALID()` are named as +not, so those "always run offline and an exclusive (Sch-M) lock is acquired for the duration". Never +online: `max` and LOB types, and any column pushing the row past 8,060 bytes. Widening a string is +free **unless the target is `MAX`**; changing nullability rewrites even when every row has a value. + +## 2. The lock, and the queue behind it + +Every `ALTER TABLE` above held a **schema modification lock, `Sch-M`**, for its whole transaction, +incompatible with every other mode, the schema stability lock a plain `SELECT` takes included. +Behind an uncommitted `Sch-M` held 3 seconds, a default read committed snapshot read, an explicit +`SNAPSHOT` isolation read and a `WITH (NOLOCK)` read each waited 3.10 s on `LCK_M_SCH_S`. + +**Then the part that turns a fast change into an outage.** With one 10 second reporting query in +flight, a `SELECT` arriving after the `ALTER` is blocked by the *pending* `ALTER`, not the slow +query: + +| Session | Own work | Waited | +|---|---|---| +| reporting query | 10 s | 10.0 s | +| `ALTER TABLE ... ADD c int NULL` | ~2 ms | **9.0 s**, `LCK_M_SCH_M` | +| three trivial `SELECT`s issued after it | ~1 ms each | **8.0 s each**, blocked by the `ALTER` | + +A 2 millisecond change stalled the table for 8 seconds, and nothing in the migration output says so. + +## 3. The three settings that decide whether anyone notices + +**`WITH (ONLINE = ON)`.** Same table: `ALTER COLUMN w2 bigint` took 8.4 s and held two concurrent +`SELECT`s **7.17 s each**; with `WITH (ONLINE = ON)` it took 3.5 s and they returned in **74 ms and +76 ms**. Not always faster in isolation, but it does not convoy: behind the same 10 second reader, later +`SELECT`s got through in 80 ms. Learn scopes it to "data type, column length or precision, +nullability, sparseness, and collation". Measured: accepted on `ALTER COLUMN`, `ALTER TABLE REBUILD`, +`CREATE INDEX`, `ALTER INDEX REBUILD`; a syntax error on `ADD` and `DROP COLUMN`; and `ALTER COLUMN` +refused with `Msg 5074`, online or not, when an index or default constraint references the column. + +**`ELEVATE_ONLINE`, which makes forgetting it impossible.** Learn: default `OFF`; `WHEN_SUPPORTED` +runs online whatever can; `FAIL_UNSUPPORTED` **fails the statement** when it cannot, naming the +non-nullable column add as the case it refuses. It touches only statements that did not say `ONLINE`, +so it is a floor, not an override, and turns section 1's `NEWID()` default from a review question +into a refusal. + +```sql +ALTER DATABASE SCOPED CONFIGURATION SET ELEVATE_ONLINE = FAIL_UNSUPPORTED; +SELECT name, value FROM sys.database_scoped_configurations +WHERE name IN ('ELEVATE_ONLINE', 'ELEVATE_RESUMABLE'); +``` + +**`SET LOCK_TIMEOUT`.** With the same reader in flight, an offline `ALTER COLUMN` under +`SET LOCK_TIMEOUT 2000` gave up after 2.1 s with `Msg 1222`, and the queries behind it waited 1.1 s +instead of 8 s. The trap, from Learn: `Msg 1222` cancels the statement and **does not roll back its +transaction**, so a lock timeout inside `BEGIN TRAN` needs `SET XACT_ABORT ON` or the script carries +on missing a statement. + +For rebuilds only, `WAIT_AT_LOW_PRIORITY` is more precise; Learn says it "can't be used with online +`ALTER COLUMN`", the `Msg 102` measured here. `RESUMABLE = ON` requires `ONLINE = ON`, pauses to fit +a window and survives a failover, but **cannot run inside an explicit transaction**. Before writing +either, open run 4 of [the measured runs](references/alter-cost-schema-locks-and-retry.md) for a +worked pausable build, its `PAUSE`, and a rebuild that gives up rather than queue the workload. + +A paused build still costs disk and DML, and any statement needing a table level exclusive lock, +such as `INSERT ... WITH (TABLOCK)`, fails with error `10637` until resumed or `ABORT`ed. + +## 4. Expand and contract, the only shape that works here + +One database, which cannot be swapped, must satisfy the code either side of the change, so "blue +green database deploy" reduces to this. **Expand** adds, never alters or renames, nullable or with a +runtime constant default, so it lands in milliseconds. **Migrate** +backfills in bounded batches, in its own deployment. **Contract** drops the old shape only after +every instance referencing it is gone: measured, after another session dropped a column, a +connection on the old shape got `Msg 207, Invalid column name` on its next write while its +`SELECT *` kept working. The failure is per statement and hits instances not yet replaced. + +**Never rename in place.** `sp_rename` on a column raised no error, only informational message +15477 at severity 10, "Caution: Changing any part of an object name could break scripts and +stored procedures", which a runner logging only exceptions discards. A view and a procedure over the +column kept existing and failed with `Msg 207` **when next called**, while +`sys.sql_expression_dependencies` still resolved both with `is_ambiguous = 0`. + +## 5. Never migrate on application startup, measured + +Eight instances starting at once, each running four steps: add a column, backfill a value, create an +index, record the version. + +| Script shape | Outcome | +|---|---| +| No guards, no transaction | **7 of 8 failed with `Msg 2705`**, duplicate column name; one succeeded | +| `IF NOT EXISTS` guards, no transaction | All 8 passed the guards; backfill ran **8 times**, 4000 where 500 was correct. Only error: `Msg 2627` on the version row, **after** the data was wrong | +| The same guards, one transaction | 7 of 8 rolled back. Data correct, 7 instances reported failure | +| `sp_getapplock` first | **1 applied, 7 skipped, 0 errors** | + +In row two the guards worked as written and the data was still wrong, because +`UPDATE ... SET tier = tier + 1` has no `IF NOT EXISTS` form; what surfaced was a duplicate key on +the history table, reading as a harmless double record, while the wrong number sat in a column nobody +watched. Row one fails visibly, the better failure. Row two ships. + +Startup migration also leaves schema altering permission on the runtime identity for good and makes +every restart a potential schema change. Migrations are a deployment step with their own identity, +carried by `sql-database-projects` and `github-actions-for-sql`. + +**If a runner must survive concurrent starts**, hold an application lock for the whole migration. +Learn's return codes: `0` granted, `1` granted after waiting, `-1` timed out, `-2` cancelled, `-3` +deadlock victim, `-999` a bad call. `@LockOwner = 'Transaction'` must run inside one. + +```sql +SET XACT_ABORT ON; +BEGIN TRAN; +DECLARE @rc int; +EXEC @rc = sys.sp_getapplock @Resource = 'schema-migrations', @LockMode = 'Exclusive', + @LockOwner = 'Transaction', @LockTimeout = 30000; +IF @rc < 0 THROW 50000, 'Could not acquire the migration lock', 1; +-- re-read the history table HERE, inside the lock, then apply +COMMIT; +``` + +## 6. Retry and migrations, the hazard nothing reports + +Retry is not optional on Azure SQL Database, so every path carries a retry policy, the migration's +included; `connect-to-azure-sql` owns that doctrine. **A retriable unit will be run more than +once.** Reproduced: the same guarded four steps, faulted once after the +backfill committed and before the version row was written, under a policy treating a broken +connection as transient. + +```text +attempt 1: FAILED, "Connection may have been terminated by the server", classified transient +attempt 2: applied, no error +runner reported success: True +final state: version 1 recorded, backfill applied TWICE (1000 where 500 was correct) +``` + +No exception, no warning, and the history table says applied. The same four steps in **one +transaction**, faulted at the same point, ended correct. + +So: **either the whole migration is one transaction, or every statement in it is idempotent, data +statements included.** A guard on the DDL and none on the backfill is the shape that reproduced. +Three limits decide where it is split: + +- `ALTER DATABASE ... SET` is refused inside a multi statement transaction with `Msg 226`. +- `RESUMABLE = ON` cannot run inside an explicit transaction at all. +- `CREATE INDEX WITH (ONLINE = ON)` **is** accepted there, and Learn warns why you should still not: + its final `S` or `Sch-M` lock is then held until commit, rebuilding section 2's convoy by hand. + +## 7. Idempotent scripts on this engine + +The syntax an agent reaches for from another database mostly does not exist here. Verified: + +| Written | Result | +|---|---| +| `CREATE TABLE IF NOT EXISTS` | `Msg 156`, syntax error. Does not exist | +| `CREATE INDEX IF NOT EXISTS` | `Msg 156`, same | +| `ALTER TABLE ... ADD IF NOT EXISTS` | `Msg 156`, same | +| `ALTER TABLE ... DROP COLUMN IF EXISTS` | **works**, and `DROP CONSTRAINT IF EXISTS` too | +| `IF OBJECT_ID(...) IS NULL CREATE VIEW ...`, same for `CREATE SCHEMA` | `Msg 156`. Cannot follow `IF` | +| `SELECT 1; CREATE VIEW ...` | `Msg 111`, must be first in the batch | +| `CREATE OR ALTER TABLE` | `Msg 156`. Only programmability objects have this | +| `IF ... EXEC sp_executesql N'CREATE VIEW ...'` | works | +| `CREATE OR ALTER VIEW` | works, and needs no guard | +| `DROP TABLE IF EXISTS` | works | + +Rows three and four are the asymmetry: contract is idempotent for free, expand is not. So expand is +guarded, anything that must be first in its batch goes inside `sp_executesql`, the backfill carries +its own predicate, and the history row is written in the same transaction as the change, because +that gap is where a retry re-applies. + +```sql +SET XACT_ABORT ON; +BEGIN TRAN; + +-- Expand: guarded, there is no ADD IF NOT EXISTS. +IF NOT EXISTS (SELECT 1 FROM sys.columns + WHERE object_id = OBJECT_ID('dbo.customers') AND name = 'region') + ALTER TABLE dbo.customers ADD region nvarchar(20) NULL; + +-- Backfill: unguarded, the predicate makes a second run a no operation. +UPDATE TOP (5000) dbo.customers SET region = N'unknown' WHERE region IS NULL; + +-- Contract: the constraint name is resolved, never typed. +DECLARE @df sysname = (SELECT d.name FROM sys.default_constraints AS d + JOIN sys.columns AS c ON c.object_id = d.parent_object_id + AND c.column_id = d.parent_column_id + WHERE d.parent_object_id = OBJECT_ID('dbo.customers') AND c.name = 'legacy'); +IF @df IS NOT NULL EXEC sys.sp_executesql N'ALTER TABLE dbo.customers DROP CONSTRAINT ' + @df; +ALTER TABLE dbo.customers DROP COLUMN IF EXISTS legacy; + +INSERT INTO dbo.__migrations (version) VALUES (7); +COMMIT; +``` + +**A guard is a race, not a guard.** Every `IF NOT EXISTS` reads the catalog, releases the lock, then +acts, so two runners both read absent and both act; section 5 measured eight of eight. Only +serialisation makes a script safe to run twice. + +**Names you did not choose will break the script.** An inline default constraint gets a generated +name that differs per environment, and `DROP COLUMN qty` failed with `Msg 5074, The object +'DF__parts__qty__123EB7A3' is dependent on column 'qty'`. Hence the lookup above. Re-run errors +meaning a script is not idempotent: `Msg 2705`, `Msg 2714`, `Msg 1913`, and `Msg 2627` on the +history table. + +## 8. Rollback is a forward migration + +- **A down script that drops the column loses every value written since the up ran**, and the up + shipped because something started writing to that column. +- **`DROP COLUMN` is metadata only, so the space is not reclaimed and the row budget is not + released.** Re-adding the dropped `char(4000)` column, which is what "roll back, fix, roll + forward" does, failed with **`Msg 1701`**, minimum row size 11011 against a maximum of 8060 bytes, + counting a column no longer in `sys.columns`. `ALTER TABLE ... REBUILD` reclaimed it and the + re-add succeeded; page counts and the full message are in the reference. +- **The reversing statement is often the expensive one.** Widening to `varchar(max)` cost 7.3 s; + narrowing back is another rewrite and fails outright if any row no longer fits. + +Plan so that not proceeding is free. Expand is reversible by being ignored; contract is the +irreversible step, gated on the old code being gone. After it, the undo is a new forward migration +against the state the database is in. + +## Check it worked + +Run the migration through `sqlcmd` with **`-m-1`**, no space between flag and value. `-b` sets a +non-zero exit only above severity 10, so alone it reports success on the section 4 `sp_rename` +caution, and without `-m-1` that message prints with no `Msg` number. **`-m-1` is an ODBC `sqlcmd` +instruction**, the 18.x build from `mssql-tools18`; go-sqlcmd 1.10.0 prints no `Msg` header on a +severity 10 message at any `-m` value (measured 2026-09-05), so there the caution arrives unnumbered. `build-app-on-azure-sql` tells the builds apart: + +```bash +export SQLCMDPASSWORD='' +SQLCMD="sqlcmd -S .database.windows.net,1433 -d -U -C -m-1" +$SQLCMD -b -i migration.sql -o migration.out +echo "exit $?" +grep -nE '^Msg |Caution:' migration.out +``` + +Expect `exit 0` and no output from `grep`. A `Msg 15477` line is a rename nobody asked for. Then ask +the database what happened, as `migration-check.sql`: + +```sql +-- 1. The change landed and the history agrees. Expect applied = 1, a non-NULL length. +SELECT (SELECT COUNT(*) FROM dbo.__migrations WHERE version = 7) AS applied, + COL_LENGTH('dbo.customers', 'region') AS region_bytes; + +-- 2. Nothing left half applied by a retry or a bounded batch. Expect zero rows. +SELECT TOP (5) id, region FROM dbo.customers WHERE region IS NULL; + +-- 3. No transaction still open, and the lock timeout this session ran under. Expect 0. +SELECT @@TRANCOUNT AS open_transactions, @@LOCK_TIMEOUT AS lock_timeout_ms; + +-- 4. No resumable index build left paused. Expect zero rows. +SELECT OBJECT_NAME(object_id) AS table_name, name, state_desc, percent_complete +FROM sys.index_resumable_operations; +``` + +```bash +# same shell as above, $SQLCMD and SQLCMDPASSWORD already set +$SQLCMD -W -i migration-check.sql +``` + +Check 3 is the one people skip: a non-zero `@@TRANCOUNT` after a script that set a lock timeout is +the section 3 `Msg 1222` trap, with the migration holding locks right now. A row in check 4 is a +paused build still blocking exclusive table locks with error `10637`. + +## Do not + +- Do not call an `ALTER TABLE` cheap without saying which kind, assume snapshot isolation protects + readers from one, or run an offline `ALTER COLUMN` or index build on a live table when the online + form is accepted. +- Do not set a lock timeout inside a transaction without `SET XACT_ABORT ON`, or put a + `RESUMABLE = ON` build inside the migration transaction; it is refused. +- Do not treat `IF NOT EXISTS` as idempotence, migrate from application startup, or leave a backfill + in a retried unit without a predicate making the second run a no operation. +- Do not `sp_rename` anything referenced, treat a down script as the rollback plan, or use this skill + to choose a migration tool: it is the doctrine under all of them. + +## References + +- Open [the measured runs](references/alter-cost-schema-locks-and-retry.md) for the statement + behind a number above, the lock traces, the retry fault injection, or the two claims that did not + hold. Read it before changing any figure here. +- [ALTER TABLE](https://learn.microsoft.com/sql/t-sql/statements/alter-table-transact-sql) before an + online alteration: the runtime constant rule and the `ONLINE` restrictions. +- [Guidelines for online index operations](https://learn.microsoft.com/sql/relational-databases/indexes/guidelines-for-online-index-operations) + before a resumable build, and when error 10637 blocks a `TABLOCK` insert. +- [ALTER DATABASE SCOPED CONFIGURATION](https://learn.microsoft.com/sql/t-sql/statements/alter-database-scoped-configuration-transact-sql) + before setting `ELEVATE_ONLINE` or `ELEVATE_RESUMABLE` on a shared database. +- [Transaction locking and row versioning](https://learn.microsoft.com/sql/relational-databases/sql-server-transaction-locking-and-row-versioning-guide) + for the lock compatibility matrix and section 3's `LOCK_TIMEOUT` rollback rule. +- [sys.sp_getapplock](https://learn.microsoft.com/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql) + for return codes and lock owner semantics. +- This skill decides whether a change is safe; `sql-database-projects` and `github-actions-for-sql` + carry it, `ef-core-azure-sql` and `sqlalchemy-azure-sql` are per tool workflows, the rest are in + `skill.spec.jsonc`. Also `design-azure-sql-schema`, `connect-to-azure-sql` for retry, and + `deploy-app-to-azure` for rolling the instances. diff --git a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/references/alter-cost-schema-locks-and-retry.md b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/references/alter-cost-schema-locks-and-retry.md new file mode 100644 index 0000000..3af6df6 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/references/alter-cost-schema-locks-and-retry.md @@ -0,0 +1,360 @@ +# Verified behaviour: every run behind the migration doctrine + +## Contents + +- [The engine and the fixture](#the-engine-and-the-fixture) +- [Run 1: metadata only versus rewrite](#run-1-metadata-only-versus-rewrite) +- [Run 2: does a schema lock block a snapshot reader](#run-2-does-a-schema-lock-block-a-snapshot-reader) +- [Run 3: the convoy behind a pending schema change](#run-3-the-convoy-behind-a-pending-schema-change) +- [Run 4: lock timeout, online, and wait at low priority](#run-4-lock-timeout-online-and-wait-at-low-priority) +- [Run 5: eight instances migrating at startup](#run-5-eight-instances-migrating-at-startup) +- [Run 6: a migration under a retry policy, faulted mid script](#run-6-a-migration-under-a-retry-policy-faulted-mid-script) +- [Run 7: what parses as an idempotent script](#run-7-what-parses-as-an-idempotent-script) +- [Run 8: rename, contract, and drop then re-add](#run-8-rename-contract-and-drop-then-re-add) +- [What did not hold](#what-did-not-hold) +- [Reproducing this](#reproducing-this) + +## The engine and the fixture + +All runs 2026-08-28, on one engine, one database. + +```text +SERVERPROPERTY('EngineEdition') 5 +SERVERPROPERTY('Edition') SQL Azure +compatibility_level 170 +is_read_committed_snapshot_on 1 +snapshot_isolation_state_desc ON +``` + +Fixture, in `appdb`: + +```sql +CREATE TABLE dbo.orders ( + id int IDENTITY(1,1) PRIMARY KEY, + customer_id int NOT NULL, + status varchar(20) NOT NULL, + amount decimal(12,2) NOT NULL, + note nvarchar(100) NULL, + created_at datetime2 NOT NULL DEFAULT SYSUTCDATETIME() +); +-- 2,000,000 rows, 230 MB reserved +``` + +Concurrency was driven by separate client connections, one per thread, not by nested batches. + +## Run 1: metadata only versus rewrite + +Each statement ran inside `BEGIN TRAN`, was timed, then +`sys.dm_tran_database_transactions.database_transaction_log_bytes_used` was read for +`CURRENT_TRANSACTION_ID()` before `COMMIT`. + +| Statement | Time | Log bytes | Log records | +|---|---|---|---| +| `ALTER TABLE dbo.orders ADD c1 int NULL` | 3.6 ms | 1,240 | 11 | +| `ADD c2 int NOT NULL CONSTRAINT df_c2 DEFAULT (0)` | 1.7 ms | 2,200 | 19 | +| `ADD c3 uniqueidentifier NOT NULL DEFAULT (NEWID())` | **4,585 ms** | **448,002,068** | **2,000,018** | +| `ADD c4 datetime2 NOT NULL DEFAULT (SYSUTCDATETIME())` | 1.5 ms | 2,220 | 19 | +| `ADD c5 varchar(max) NULL` | 1.6 ms | 2,336 | 23 | +| `ALTER COLUMN status varchar(50) NOT NULL` (from 20) | 1.0 ms | 1,068 | 10 | +| `ALTER COLUMN status varchar(max) NOT NULL` | **7,260 ms** | **572,400,240** | **2,000,018** | +| `ALTER COLUMN note2 nvarchar(50)` (from `varchar(50)`) | **5,070 ms** | **426,473,648** | **2,000,027** | +| `ALTER COLUMN customer_id bigint NOT NULL` | **6,053 ms** | **508,588,372** | **2,000,018** | +| `ALTER COLUMN note nvarchar(100) NOT NULL` (from `NULL`) | **14,508 ms** | **661,262,280** | **2,000,023** | +| `ADD CONSTRAINT ck_amt CHECK (amount >= 0)` | 521 ms | 1,704 | 17 | +| `WITH NOCHECK ADD CONSTRAINT ck_amt2 CHECK (...)` | 1.5 ms | 1,388 | 12 | +| `DROP COLUMN c5` | 2.9 ms | 1,224 | 12 | +| `CREATE INDEX ix_off ON dbo.orders(customer_id)` | 628 ms | 37,714,632 | 4,510 | +| `CREATE INDEX ix_on ON dbo.orders(amount) WITH (ONLINE=ON)` | 3,489 ms | 1,712 | 12 | + +`sys.dm_tran_locks` sampled from a second session while each transaction was open showed +`OBJECT` / `Sch-M` for every `ALTER TABLE`, and `HOBT` / `Sch-M` plus `OBJECT` / `Sch-S` for the +index builds. + +A first pass of this run reported near zero log bytes for most statements. The cause was the client +being opened with autocommit off, so an outer driver transaction wrapped the explicit one and +rolled everything back at close. The table above is the corrected run, with autocommit on and +`BEGIN TRAN` and `COMMIT` issued explicitly. Worth recording, because the first pass looked +plausible. + +## Run 2: does a schema lock block a snapshot reader + +Session A: `BEGIN TRAN; ALTER TABLE dbo.orders ADD z1 int NULL;` held open 3 seconds, then +committed. Three readers started 0 ms later, one per connection. + +```text +sys.dm_exec_requests, 2 s in: + 52 LCK_M_SCH_S wait_time 2047 blocked by 51 + 53 LCK_M_SCH_S wait_time 2044 blocked by 52 + 54 LCK_M_SCH_S wait_time 2042 blocked by 52 + +elapsed at the client: + default read committed snapshot 3,098 ms + SET TRANSACTION ISOLATION LEVEL SNAPSHOT, explicit tran 3,102 ms + SELECT ... WITH (NOLOCK) 3,100 ms +``` + +All three waited the full hold. The `ALTER` itself was a 2 ms metadata only change; what blocked +was the transaction holding `Sch-M`, not the work. + +## Run 3: the convoy behind a pending schema change + +Session A: `BEGIN TRAN; SELECT TOP 1 id FROM dbo.orders WITH (REPEATABLEREAD); WAITFOR DELAY +'00:00:10'; COMMIT;` +Session B, 1 s later: `ALTER TABLE dbo.orders ADD z1 int NULL`. +Sessions C, D, E, 1 s after that: `SELECT TOP 1 id FROM dbo.orders`. + +```text +sys.dm_exec_requests, 3 s in: + 51 LCK_M_SCH_M blocked by 54 ALTER TABLE dbo.orders ADD z1 int NULL + 52 LCK_M_SCH_S blocked by 51 SELECT TOP 1 id FROM dbo.orders + 53 LCK_M_SCH_S blocked by 51 SELECT TOP 1 id FROM dbo.orders + 55 LCK_M_SCH_S blocked by 51 SELECT TOP 1 id FROM dbo.orders + +elapsed: + long reader 10,011 ms + ALTER (2 ms work) 9,017 ms + tiny SELECT #0 8,025 ms + tiny SELECT #1 8,020 ms + tiny SELECT #2 8,023 ms +``` + +The three trivial reads are blocked by the `ALTER`, not by the long reader. This is what turns a +metadata only change into a full stall. + +## Run 4: lock timeout, online, and wait at low priority + +**`SET LOCK_TIMEOUT 2000` in front of an offline `ALTER COLUMN`**, same 10 second reader in flight: + +```text +the ALTER ERR Msg 1222, Lock request time out period exceeded, 2,099 ms +tiny SELECT #0 1,095 ms +tiny SELECT #1 1,077 ms +``` + +Compare the 8,020 ms in run 3 with no timeout set. + +**Online versus offline on the same column**, `w2`, no index and no default constraint on it, so +neither form is refused. Two concurrent `SELECT TOP 1` readers started 1.2 s into the change: + +| Statement | Its own time | Reader 1 | Reader 2 | +|---|---|---|---| +| `ALTER COLUMN w2 bigint NULL` | 8,369 ms | 7,169 ms | 7,169 ms | +| `ALTER COLUMN w2 int NULL WITH (ONLINE = ON)` | 4,085 ms | 75 ms | 95 ms | +| `ALTER COLUMN w2 bigint NULL WITH (ONLINE = ON)` | 3,513 ms | 74 ms | 76 ms | + +`sys.dm_exec_requests` showed `LCK_M_SCH_S` waiters during the offline run and none during either +online run. + +**Online does not convoy.** With the 10 second reader in flight and an online `ALTER COLUMN` +waiting behind it, `SELECT`s arriving after the `ALTER` returned in 80 and 82 ms, against 8,020 ms +for the offline form in run 3. + +**Where `ONLINE = ON` is accepted:** + +| Statement | Result | +|---|---| +| `ALTER TABLE ... ALTER COLUMN ... WITH (ONLINE = ON)` | accepted | +| `ALTER TABLE ... REBUILD WITH (ONLINE = ON)` | accepted, 2,961 ms | +| `CREATE INDEX ... WITH (ONLINE = ON)` | accepted, inside and outside an explicit transaction | +| `ALTER TABLE ... ADD col ... WITH (ONLINE = ON)` | `Msg 102`, syntax error | +| `ALTER TABLE ... DROP COLUMN c1 WITH (ONLINE = ON)` | `Msg 156`, syntax error | +| `ALTER COLUMN` on a column an index references | `Msg 5074`, online or not | +| `ALTER COLUMN` on a column a default constraint references | `Msg 5074`, online or not | + +**`WAIT_AT_LOW_PRIORITY`:** + +| Statement | Result | +|---|---| +| `ALTER INDEX ... REBUILD WITH (ONLINE = ON (WAIT_AT_LOW_PRIORITY (MAX_DURATION = 1 MINUTES, ABORT_AFTER_WAIT = SELF)))` | parses and runs | +| `ALTER TABLE ... REBUILD WITH (ONLINE = ON (WAIT_AT_LOW_PRIORITY (...)))` | parses and runs | +| `ALTER TABLE ... ALTER COLUMN ... WITH (ONLINE = ON (WAIT_AT_LOW_PRIORITY (...)))` | **`Msg 102`, syntax error** | + +**The two forms written out.** A resumable build, which requires `ONLINE = ON` and cannot run inside +an explicit transaction, pauses to fit a window and is watched in `sys.index_resumable_operations`. +A rebuild with `WAIT_AT_LOW_PRIORITY` gives up rather than queueing the workload behind itself; it +is for rebuilds only, per the table above. + +```sql +-- Pausable build. Not inside BEGIN TRAN. +CREATE INDEX ix_orders_customer ON dbo.orders (customer_id) +WITH (ONLINE = ON, RESUMABLE = ON, MAX_DURATION = 20 MINUTES); +SELECT name, state_desc, percent_complete FROM sys.index_resumable_operations; +ALTER INDEX ix_orders_customer ON dbo.orders PAUSE; + +-- Rebuild that gives up rather than queueing the workload behind itself. +ALTER INDEX ix_orders_customer ON dbo.orders REBUILD +WITH (ONLINE = ON (WAIT_AT_LOW_PRIORITY (MAX_DURATION = 1 MINUTES, ABORT_AFTER_WAIT = SELF))); +``` + +## Run 5: eight instances migrating at startup + +Fixture reset before each mode: `dbo.__migrations(version int PRIMARY KEY, applied_at)` and +`dbo.customers(id, name, tier int NOT NULL DEFAULT 0)` with 500 rows, all `tier = 0`. + +Each of eight threads, on its own connection, checked the history table and, if version 1 was +absent, ran four statements with a 50 ms pause between them: + +```text +1 ALTER TABLE dbo.customers ADD region nvarchar(20) NULL +2 UPDATE dbo.customers SET tier = tier + 1 +3 CREATE INDEX ix_cust_tier ON dbo.customers(tier) +4 INSERT INTO dbo.__migrations (version) VALUES (1) +``` + +Correct final state is version count 1, `region` present, index present, `SUM(tier) = 500`. + +| Mode | Decided to apply | Errors | Final `SUM(tier)` | +|---|---|---|---| +| no guards, no transaction | 8 | 7 x `Msg 2705` on statement 1 | 500 | +| `IF NOT EXISTS` guards on 1 and 3, no transaction | 8 | 7 x `Msg 2627` on statement 4 | **4000** | +| the same guards, all four inside one transaction | 8 | 7 x `Msg 2627` on statement 4 | 500 | +| `sp_getapplock` Exclusive, Transaction owner, before the check | **1 applied, 7 skipped** | **0** | 500 | + +Exact messages: + +```text +2705 Column names in each table must be unique. Column name 'region' in table + 'dbo.customers' is specified more than once. +2627 Violation of PRIMARY KEY constraint 'PK____migrat__...'. Cannot insert duplicate key + in object 'dbo.__migrations'. The duplicate key value is (1). +``` + +Row two is the one that matters. The guards did their job, every runner passed them, and the +un-guardable `UPDATE` ran eight times. The only error surfaced on the history insert, after the +data was already wrong. + +Related re-run errors, captured separately: `Msg 2714` for a `CREATE TABLE` or a named constraint +that already exists, `Msg 1913` for an index name that already exists. + +## Run 6: a migration under a retry policy, faulted mid script + +Migration: the same guarded four steps, with `WAITFOR DELAY '00:00:03'` inserted before the history +insert to stand in for the rest of a real migration. Runner: retry up to three attempts, treating a +broken connection or a documented transient error number as retriable, which is the shape every +data access guide teaches. Fault: a second session issued `KILL ` 1.5 s into attempt 1, which +lands inside the `WAITFOR`, that is after the backfill committed and before the history row. + +**No transaction:** + +```text +attempt 1: FAILED err=None [TRANSIENT -> retried] + [Microsoft][ODBC Driver 18 for SQL Server]Unspecified error occurred on SQL Server. + Connection may have been terminated by the server. (0) +attempt 2: applied, no error +runner reported success: True +final: versions 1, region present, SUM(tier) = 1000 <- correct is 500 +``` + +**Same script, all steps inside one transaction, same fault at the same point:** + +```text +attempt 1: FAILED, same transient classification +attempt 2: applied, no error +runner reported success: True +final: versions 1, region present, SUM(tier) = 500 <- correct +``` + +The hazard reproduces. Nothing raised, nothing logged, exit zero, data doubled. Two caveats stated +plainly: the fault was injected with `KILL` rather than by waiting for a real transient fault, and +what the client saw was a broken connection with no SQL error number, surfaced by the driver as +native error 0. That is the class of failure transient fault policies exist to retry, so the +classification in the harness is the same one a production policy would make, but it is not proof +that a specific library's classifier would agree. + +**Transaction limits found while testing this**, each attempted inside `BEGIN TRAN`: + +| Statement | Inside a transaction | +|---|---| +| `CREATE INDEX`, offline or `WITH (ONLINE = ON)` | accepted | +| `ALTER TABLE ... REBUILD WITH (ONLINE = ON)` | accepted | +| `ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 160` | **`Msg 226`**, not allowed within a multi statement transaction | + +## Run 7: what parses as an idempotent script + +| Written | Result | +|---|---| +| `CREATE TABLE IF NOT EXISTS dbo.t (id int)` | `Msg 156`, incorrect syntax near the keyword 'IF' | +| `CREATE INDEX IF NOT EXISTS ix ON ...` | `Msg 156` | +| `ALTER TABLE dbo.parts ADD IF NOT EXISTS zz int NULL` | `Msg 156` | +| `CREATE OR ALTER TABLE dbo.t (id int)` | `Msg 156`, incorrect syntax near the keyword 'TABLE' | +| `IF OBJECT_ID('dbo.v_x') IS NULL CREATE VIEW dbo.v_x AS ...` | `Msg 156`, incorrect syntax near the keyword 'VIEW' | +| `SELECT 1; CREATE VIEW dbo.v_y AS ...` | `Msg 111`, `CREATE VIEW` must be the first statement in a query batch | +| `IF SCHEMA_ID('app') IS NULL CREATE SCHEMA app` | `Msg 156`, incorrect syntax near the keyword 'SCHEMA' | +| `IF OBJECT_ID('dbo.v_x') IS NULL EXEC sp_executesql N'CREATE VIEW dbo.v_x AS ...'` | works | +| `CREATE OR ALTER VIEW dbo.v_x AS ...` | works, inside and outside a transaction | +| `DROP TABLE IF EXISTS dbo.t` | works | + +## Run 8: rename, contract, and drop then re-add + +**`sp_rename` on a column.** `EXEC sp_rename 'dbo.parts.sku', 'part_number', 'COLUMN'` raised no +error. The driver received one informational message: + +```text +[01000] (15477) Caution: Changing any part of an object name could break scripts and + stored procedures. +``` + +A view and a procedure written over `sku` then failed with `Msg 207, Invalid column name 'sku'` at +the next `SELECT` and the next `EXEC`. `sys.sql_expression_dependencies` still returned both +referencing objects against `dbo.parts` with `is_ambiguous = 0`, so the dependency metadata gives +no signal that either is broken. + +**Contract, from the point of view of an instance that has not been replaced.** A connection +inserted successfully, another session ran `ALTER TABLE dbo.parts2 DROP COLUMN legacy_note`, then +the same connection: + +```text +its very next INSERT naming legacy_note Msg 207, Invalid column name 'legacy_note' +its SELECT * still works, with one fewer column +``` + +**Drop then re-add.** Table `dbo.wide2(id int IDENTITY PRIMARY KEY, a char(4000) NOT NULL, +b char(3000) NOT NULL)`, 200 rows. + +```text +used_page_count before DROP COLUMN a 202 +used_page_count after DROP COLUMN a 202 <- nothing reclaimed +ALTER TABLE dbo.wide2 ADD a char(4000) NOT NULL DEFAULT '' + Msg 1701: Creating or altering table 'wide2' failed because the minimum row size would be + 11011, including 7 bytes of internal overhead. This exceeds the maximum allowable table + row size of 8060 bytes. +ALTER TABLE dbo.wide2 REBUILD +used_page_count after REBUILD 102 +re-add after REBUILD succeeded +``` + +11011 is the row size counting the dropped `char(4000)`, which no longer appears in +`sys.columns`. `sys.system_internals_partition_columns`, the catalog view that would show it, +returned `Msg 208, Invalid object name` here, so the space cannot be inspected directly. + +**Unnamed default constraint.** A column declared `qty int NOT NULL DEFAULT (0)` produced +`DF__parts__qty__123EB7A3`. `ALTER TABLE dbo.parts DROP COLUMN qty` failed with +`Msg 5074, The object 'DF__parts__qty__123EB7A3' is dependent on column 'qty'`. Looking the name up +in `sys.default_constraints` and dropping it through `sp_executesql` succeeded. The generated suffix +differs per environment, so a script naming it literally works where it was written and nowhere +else. + +## What did not hold + +- **"Online index operations cannot run inside an explicit transaction."** Expected, and false + here. `CREATE INDEX ... WITH (ONLINE = ON)` and `ALTER TABLE ... REBUILD WITH (ONLINE = ON)` both + ran inside `BEGIN TRAN` and committed. The claim was dropped rather than softened. +- **`WAIT_AT_LOW_PRIORITY` as a general answer to the convoy.** It is rejected with `Msg 102` on + `ALTER TABLE ... ALTER COLUMN`, so it covers rebuilds only. `SET LOCK_TIMEOUT` is the setting that + applies to every statement. +- **A first attempt at run 1** reported near zero log bytes for almost every statement, which would + have made every alteration look metadata only. Cause was an outer driver transaction rolling each + case back. Corrected before anything was written down. +- **The `sp_rename` caution as a warning the operator sees.** It is class 01, informational. The + client library surfaced it in a messages collection and raised nothing. A runner that logs only + exceptions logs nothing. + +## Reproducing this + +Everything above needs one Azure SQL Database engine, a table large enough that a rewrite is +visible, and two or more client connections. Two sessions are enough for runs 2, 3, 4 and 8, and +eight for run 5. The measurements that matter are wall clock at the client, `wait_type` and +`blocking_session_id` from `sys.dm_exec_requests` sampled from an uninvolved session, and +`database_transaction_log_bytes_used` from `sys.dm_tran_database_transactions` read before the +commit. Open every session with autocommit on and issue `BEGIN TRAN` explicitly, or the driver's +own transaction will silently undo the case being measured. diff --git a/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/skill.spec.jsonc new file mode 100644 index 0000000..307b7c4 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/schema-migrations-safely/skill.spec.jsonc @@ -0,0 +1,348 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "schema-migrations-safely", + "domain": "schema-change-and-cicd", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked whether a schema change is safe to apply to a live Azure SQL Database, an agent answers with expand and contract, adds IF NOT EXISTS guards, and calls the script idempotent, and all three of those beliefs are wrong in ways nothing reports. Measured on 2026-08-28 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure' at compatibility level 170 with read committed snapshot isolation on, over a 2,000,000 row table: ALTER TABLE is not uniformly cheap and the expensive statements are not the ones that look expensive, since adding a NOT NULL column with a constant default took 1.7 ms and 2.2 KB of log while adding one with a NEWID() default took 4.6 s and 448 MB, widening varchar(20) to varchar(50) took 1.0 ms while widening to varchar(max) took 7.3 s and 572 MB, and changing NULL to NOT NULL took 14.5 s and 661 MB. The schema modification lock those statements hold is the one thing snapshot isolation does not protect a reader from: a default read, an explicit SNAPSHOT read and a NOLOCK read all waited 3.10 s on LCK_M_SCH_S behind an uncommitted 2 ms ALTER, and worse, when one 10 second query was already in flight the 2 ms ALTER waited 9.0 s and three trivial SELECTs issued after it waited 8.0 s each blocked by the ALTER rather than by the slow query, which is how a metadata only change becomes an outage. The same change offline held two readers for 7.17 s where WITH (ONLINE = ON) let them through in 74 ms. A guard protects the statement it wraps and nothing else: eight instances starting at once all passed the same IF NOT EXISTS checks and applied the same backfill eight times, ending at 4000 where 500 was correct, and the only error raised was Msg 2627 on the history row after the data was already wrong, while the unguarded version failed loudly with Msg 2705 on seven of eight and left the data correct. The same guarded script under a transient fault retry policy, faulted once after the backfill committed and before the history row was written, re-ran from the top, doubled the backfill and returned success with no exception, no warning and no log line, and wrapping the four statements in one transaction was what made it correct. Rollback is not the inverse: DROP COLUMN reclaimed nothing, 202 pages before and after, and re-adding a char(4000) column failed with Msg 1701 for a column that no longer exists. What being wrong costs is that the changes which stall a production database are indistinguishable from the ones that do not until they run at production row count, and the concurrency and retry failures are silent or surface attributed to the wrong statement.", + "value_probe": { + "gate": "The answer treats each existence check as protecting only the statement it wraps, so it reaches the fact that this script can apply its effects more than once, and it decides the blocking question from what the statements do to the rows rather than from the isolation level the database runs at.", + "task": "We ship database changes with the application, and this one adds a column to a table of about two million rows, backfills that column, and writes a row to our own migrations history table. Every statement is wrapped in a check so it only runs if it has not already been applied, and our runner retries on transient failures. We deploy eight instances at once. Is this safe to run against production in the middle of the working day, or do I need a maintenance window? Tell me what would actually happen if I ran it right now.", + "states": [ + "The whole set of statements has to be one transaction, because a retry can re-run from the top after part of it already committed.", + "A schema change holds a lock that readers wait behind whatever isolation level they asked for.", + "Cost depends on whether the statement has to touch every row, so two similar looking changes can differ by orders of magnitude.", + "Several instances passing the same existence checks at the same moment can each apply the same backfill." + ], + "refutes": [ + "Guarding each statement with an existence check makes the script safe to run twice or from several instances at once.", + "Readers are unaffected because read committed snapshot isolation is on, so nothing waits behind the schema change.", + "Asking for a dirty read lets a query through while a schema change is in flight.", + "Adding a column is metadata only and therefore cheap whatever default value it carries.", + "Dropping a column and adding it back is a clean way to undo the change." + ] + }, + "posture": ["read", "write"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "sql-database-projects", + "github-actions-for-sql", + "ef-core-azure-sql", + "ef-core-migrations-azure-sql", + "prisma-azure-sql", + "prisma-migrate-azure-sql", + "sqlalchemy-azure-sql", + "alembic-azure-sql", + "flyway-and-liquibase-azure-sql", + "dacfx-and-sqlpackage", + "drift-detection", + "schema-change-review-checklist", + "design-azure-sql-schema", + "connect-to-azure-sql", + "deploy-app-to-azure", + "diagnose-blocking-and-deadlocks" + ], + "validation": { + "target": "both", + "setup": [ + "a table large enough that a row rewrite is measurable, at least a million rows", + "the ability to open at least eight client sessions against the same database concurrently", + "one session free to sample sys.dm_exec_requests and sys.dm_tran_locks while another is blocked", + "a table carrying a column declared with an inline unnamed DEFAULT constraint", + "a view or stored procedure referencing a column the migration proposes to rename or drop" + ], + "assert": [ + "every ALTER TABLE in a proposed migration is classified metadata only or table rewrite, and the classification is justified rather than asserted", + "a rewriting ALTER COLUMN or index build against a live table carries WITH (ONLINE = ON), or the answer states why there are no concurrent readers", + "no answer claims snapshot isolation or NOLOCK protects a reader from a schema change", + "the script sets a lock timeout or names a maintenance window, so a blocked change cannot queue the workload behind itself", + "no column is renamed with sp_rename, and no column is retyped or dropped in the same deployment that adds its replacement", + "the migration is not run from application startup, and the runtime identity is not the identity holding schema permission", + "the migration is either a single transaction or every statement in it, data statements included, is a no operation on its second execution", + "every backfill carries a predicate that makes re-running it harmless and runs outside the schema change", + "the history or version row is written in the same transaction as the change it records", + "any default constraint the script drops is resolved from sys.default_constraints rather than named literally", + "no down script is offered as the rollback plan, and the answer states which deployment can be skipped instead", + "a concurrency or locking claim came from two sessions rather than from a comment" + ], + "probes": [ + { + "sql": "CREATE TABLE IF NOT EXISTS dbo.probe_mig_ine (id int NOT NULL);", + "expect": { + "error": 156 + }, + "why": "The single most likely thing an agent writes when asked to make a migration re-runnable, because every other engine it has read has this form. Section 7's table says Msg 156, syntax error, does not exist, and this is that row made executable. Measured 2026-09-04 on the container at EngineEdition 5. It fails loudly rather than silently, which is the good case, but only if somebody has been told; the cost of not knowing is a migration script rewritten from scratch at the wrong moment." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_add (id int NOT NULL);\nGO\nALTER TABLE dbo.probe_mig_add ADD IF NOT EXISTS region nvarchar(20) NULL;", + "expect": { + "error": 156 + }, + "why": "The expand half of section 7's asymmetry: there is no ADD IF NOT EXISTS, so every expand step has to be wrapped in an explicit catalog check by hand. Measured 2026-09-04, Msg 156. Paired deliberately with the DROP COLUMN IF EXISTS probe below, because the conclusion the skill draws is not about either statement on its own: it is that contract is idempotent for free and expand is not, which is why the guard appears only on one half of its worked example." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_drop (id int NOT NULL, legacy int NULL);\nGO\nALTER TABLE dbo.probe_mig_drop DROP COLUMN IF EXISTS legacy;\nALTER TABLE dbo.probe_mig_drop DROP COLUMN IF EXISTS legacy;\nSELECT COUNT(*) AS remaining_columns FROM sys.columns WHERE object_id = OBJECT_ID('dbo.probe_mig_drop');", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The contract half of the same asymmetry, and it is run twice on purpose: idempotent means the second run is a no operation, not that the first one worked. One column left is the id. Measured 2026-09-04. If this ever raises, the worked example in section 7 stops being safe to re-run and the skill's claim that contract is idempotent for free is wrong." + }, + { + "sql": "SELECT 1 AS one;\nCREATE VIEW dbo.probe_mig_view AS SELECT 1 AS one;", + "expect": { + "error": 111 + }, + "why": "A different number from the four Msg 156 rows, and a different cause, which is why it is worth its own round trip: CREATE VIEW is refused not because the syntax is wrong but because it is not first in its batch. Measured 2026-09-04, Msg 111. This is the whole reason the skill routes programmability objects through sp_executesql instead of an IF, and a reader who only knows about Msg 156 will try to fix a Msg 111 by editing the statement rather than by moving it." + }, + { + "sql": "BEGIN TRAN;\nALTER DATABASE SCOPED CONFIGURATION SET ELEVATE_ONLINE = FAIL_UNSUPPORTED;\nCOMMIT;", + "expect": { + "error": 226 + }, + "why": "One of the three limits section 6 uses to decide where a migration transaction can be split, and the one the skill itself invites a reader to run: it prints ALTER DATABASE SCOPED CONFIGURATION SET ELEVATE_ONLINE in section 3 and then says in section 6 that ALTER DATABASE ... SET is refused inside a multi-statement transaction. Measured 2026-09-04, Msg 226, message text 'ALTER DATABASE SCOPED CONFIGURATION statement not allowed within multi-statement transaction.' A script that wraps the whole migration in one transaction, which is the shape section 6 recommends, cannot also set this inside it." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_res (id int NOT NULL, v int NULL);\nGO\nBEGIN TRAN;\nCREATE INDEX ix_probe_mig_res ON dbo.probe_mig_res (v) WITH (ONLINE = ON, RESUMABLE = ON);\nCOMMIT;", + "expect": { + "error": 574 + }, + "why": "The second of the three limits, and the skill states the rule without a number, so this probe supplies one that was measured rather than remembered: Msg 574, 'RESUMABLE INDEX statement cannot be used inside a user transaction', measured 2026-09-04 on the container. It matters because section 6's recommended shape is one transaction around the whole migration, and a resumable build is the one step that cannot go inside it, so the doctrine and the technique collide and the reader has to be told which gives way." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_walp (id int NOT NULL, w int NULL);\nGO\nALTER TABLE dbo.probe_mig_walp ALTER COLUMN w bigint WITH (ONLINE = ON (WAIT_AT_LOW_PRIORITY (MAX_DURATION = 1 MINUTES, ABORT_AFTER_WAIT = SELF)));", + "expect": { + "error": 102 + }, + "why": "Section 3 says WAIT_AT_LOW_PRIORITY is for rebuilds only and cannot be used with an online ALTER COLUMN, and names Msg 102 as what was measured. Reproduced 2026-09-04 on the container. This is the trap of reading section 3 too fast: the two options appear a paragraph apart, both are about not queueing a workload behind a schema change, and combining them is a parse error rather than anything that explains itself." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_dfc (id int NOT NULL, qty int NOT NULL DEFAULT (0));\nGO\nALTER TABLE dbo.probe_mig_dfc DROP COLUMN qty;", + "expect": { + "error": 5074 + }, + "why": "The names-you-did-not-choose failure in section 7, which is what the constraint lookup in the worked example exists to solve. An inline default gets a generated name that differs per environment, so the contract step fails with Msg 5074 naming an object nobody wrote down. Measured 2026-09-04, message 'The object DF__probe_mig_dfc__qty__ is dependent on column qty.' The number is asserted rather than the text precisely because the generated suffix is not stable, which is the point of the claim." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_ren (id int NOT NULL, oldname int NULL);\nGO\nEXEC sp_rename 'dbo.probe_mig_ren.oldname', 'newname', 'COLUMN';", + "expect": { + "warning": 15477 + }, + "why": "The most dangerous shape in the skill, and the only probe here that asserts a message on a statement that SUCCEEDS. sp_rename renames the column, raises nothing at severity 11 or above, and emits Msg 15477 at severity 10, which a runner logging only exceptions discards and which sqlcmd prints with no number at all unless -m-1 is set. Measured 2026-09-04. A succeeds-probe would go green here and prove nothing; asserting the warning number is what makes the silence visible. If a future build raises this as an error the skill's warning becomes unnecessary, and if it stops emitting it at all the rename becomes completely silent, which is worse." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_dep (id int NOT NULL, oldc int NULL);\nGO\nCREATE VIEW dbo.probe_mig_depv AS SELECT oldc FROM dbo.probe_mig_dep;\nGO\nEXEC sp_rename 'dbo.probe_mig_dep.oldc', 'newc', 'COLUMN';\nGO\nSELECT oldc FROM dbo.probe_mig_depv;", + "expect": { + "error": 207 + }, + "why": "The consequence of the probe above, and the reason 'never rename in place' is a rule rather than a preference: the view over the renamed column keeps existing, the rename reports nothing worth catching, and the failure arrives as Msg 207 when the view is next called, which may be in production long after the migration was declared successful. Measured 2026-09-04. Kept separate from the warning probe because the two failures happen at different times and a migration runner can catch neither." + }, + { + "sql": "ALTER DATABASE SCOPED CONFIGURATION SET ELEVATE_ONLINE = FAIL_UNSUPPORTED;\nGO\nCREATE TABLE dbo.probe_mig_elev (id int NOT NULL);\nGO\nINSERT INTO dbo.probe_mig_elev (id) SELECT value FROM GENERATE_SERIES(1, 10);\nGO\nALTER TABLE dbo.probe_mig_elev ADD g uniqueidentifier NOT NULL DEFAULT NEWID();", + "expect": { + "error": 12115 + }, + "why": "Section 3's claim that FAIL_UNSUPPORTED turns section 1's NEWID() default from a review question into a refusal, made executable and given the number the skill does not carry. Measured 2026-09-04 on the container: Msg 12115, 'ADD COLUMN statement failed by the ELEVATE_ONLINE database scoped option. Column could not be added online.' NEWID() is not a runtime constant, so the add is offline and holds an exclusive lock for the whole rewrite, and this setting is the only mechanism in the skill that stops that happening by accident rather than by review. If the setting stops refusing it, the recommendation becomes advisory and the skill should say so." + }, + { + "sql": "DECLARE @rc int;\nEXEC @rc = sys.sp_getapplock @Resource = 'probe-schema-migrations', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = 1000;\nSELECT @rc AS return_code;", + "expect": { + "rows": [ + [ + -999 + ] + ] + }, + "why": "The fix section 5 recommends, run wrong. sp_getapplock with @LockOwner = 'Transaction' outside an explicit transaction does not raise: it returns -999, the code the skill lists as 'a bad call', and a runner that checks only for an exception carries on with no lock at all, which is the exact eight-instances-all-applied failure the lock was added to prevent. Measured 2026-09-04. The probe asserts the number rather than a refusal because the whole hazard is that nothing is refused." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_dup (id int NOT NULL);\nGO\nALTER TABLE dbo.probe_mig_dup ADD region nvarchar(20) NULL;\nALTER TABLE dbo.probe_mig_dup ADD region nvarchar(20) NULL;", + "expect": { + "error": 2705 + }, + "why": "What the guard the previous probes require is actually guarding against, which none of them measure: the unguarded ADD is not quietly idempotent underneath the wrapper, the second one is refused. Measured 2026-09-05 on the container at EngineEdition 5: Msg 2705, 'Column names in each table must be unique.' That is what makes the hand-written catalog check load-bearing rather than decorative, and it is also the asymmetry section 7 turns on: an unguarded expand step re-run is loud, a guarded one is silent, and a contract step is silent either way. Red if a repeated ADD succeeds, which would make the guard cosmetic and the loud-versus-silent argument wrong." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_name (id int NOT NULL, a int NOT NULL DEFAULT 0);\nGO\nALTER TABLE dbo.probe_mig_name DROP CONSTRAINT DF_probe_mig_name_a;", + "expect": { + "error": 3728 + }, + "why": "Why the skill tells an author to resolve a default constraint out of sys.default_constraints rather than naming it literally. An inline unnamed DEFAULT gets an engine-generated name carrying an unguessable hash suffix, so the conventional name a script writes by hand matches nothing and the contract step fails in whichever environment it meets first. Measured 2026-09-05: Msg 3728, \"'DF_probe_mig_name_a' is not a constraint.\" Distinct from the Msg 5074 probe above, which measures the drop being blocked by a default that IS there; this one measures a name that is not. Red if the engine ever names an inline default predictably, which would make the lookup unnecessary." + }, + { + "sql": "CREATE TABLE dbo.probe_mig_wide (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, pad char(4000) NOT NULL, pad2 char(3000) NOT NULL);\nGO\nINSERT INTO dbo.probe_mig_wide (pad, pad2) SELECT TOP 500 'x', 'y' FROM sys.all_objects;\nGO\nDECLARE @before bigint = (SELECT SUM(used_page_count) FROM sys.dm_db_partition_stats WHERE object_id = OBJECT_ID('dbo.probe_mig_wide') AND index_id IN (0, 1));\nALTER TABLE dbo.probe_mig_wide DROP COLUMN pad;\nDECLARE @after bigint = (SELECT SUM(used_page_count) FROM sys.dm_db_partition_stats WHERE object_id = OBJECT_ID('dbo.probe_mig_wide') AND index_id IN (0, 1));\nSELECT CASE WHEN @after = @before THEN 'unchanged' WHEN @after < @before THEN 'reclaimed' ELSE 'grew' END AS space;", + "expect": { + "rows": [ + [ + "unchanged" + ] + ] + }, + "why": "DROP COLUMN is a metadata operation and the space the dropped data occupied is not returned, which is the fact the next probe depends on and the reason a contract step is not free even though it is fast. Measured 2026-09-05 on the container: the same page count before and after dropping a char(4000) column across 500 rows. Red if the engine reclaims the pages, which would mean the row width is not still committed and the refusal the next probe asserts would stop happening." + }, + { + "sql": "ALTER TABLE dbo.probe_mig_wide ADD pad char(4000) NOT NULL DEFAULT 'z';", + "expect": { + "error": 1701 + }, + "why": "Rollback is not the inverse of the change, made executable, and the sharpest argument in the skill for skipping a deployment rather than running a down script. The column the previous probe dropped cannot be added back, because its space was never reclaimed and the minimum row size would exceed the page. Measured 2026-09-05: Msg 1701, a minimum row size over the 8060 maximum, for a column that no longer exists in the table. It runs against the table the previous probe left behind on purpose, because the state, not the statement, is the claim. Red if the re-add succeeds, which would make a down script a viable rollback here." + }, + { + "sql": "SELECT CASE WHEN OBJECT_DEFINITION(OBJECT_ID('dbo.probe_mig_depv')) LIKE '%oldc%' THEN 'stale' ELSE 'updated' END AS view_definition;", + "expect": { + "rows": [ + [ + "stale" + ] + ] + }, + "why": "The mechanism behind the Msg 207 probe above, kept separate because the two are read at different times and by different people. After sp_rename the dependent view's stored definition still names the old column, so nothing in the catalog was rewritten and nothing was warned about, and an author who checks that the rename 'worked' by opening the view is reading text that no longer compiles. Measured 2026-09-05 against the fixture the sp_rename probe leaves behind. Red if the definition is rewritten by the rename, which would mean the catalog does track the change and only a compiled plan was stale." + } + , + { + "sql": "CREATE TABLE dbo.probe_m_idx (id int, qty int);\nCREATE INDEX probe_m_idx_ix ON dbo.probe_m_idx (qty);\nGO\nALTER TABLE dbo.probe_m_idx DROP COLUMN qty;", + "expect": { + "error": 5074 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. An index holds the column, so a migration that drops it has to drop the index first. The index name differs per environment, which is why the skill says read the message rather than hardcode a drop. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "BEGIN TRAN;\nALTER DATABASE CURRENT SET AUTO_SHRINK OFF;\nCOMMIT;", + "expect": { + "error": 226 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. ALTER DATABASE SET cannot sit inside a multi statement transaction, so a migration runner that wraps every step in one cannot carry a database setting. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE IF NOT EXISTS dbo.probe_m_ct (id int);", + "expect": { + "error": 156 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. CREATE TABLE IF NOT EXISTS does not exist on this engine. It is the first thing a model writes when asked for an idempotent migration. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_ci (id int);\nGO\nCREATE INDEX IF NOT EXISTS probe_m_ci_ix ON dbo.probe_m_ci (id);", + "expect": { + "error": 156 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Nor does the index form. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_at (id int);\nGO\nALTER TABLE dbo.probe_m_at ADD IF NOT EXISTS z int;", + "expect": { + "error": 156 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Nor the ALTER TABLE ADD form. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "IF OBJECT_ID('dbo.probe_m_view') IS NULL CREATE VIEW dbo.probe_m_view AS SELECT 1 AS x;", + "expect": { + "error": 156 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The guard a developer writes instead cannot work either, because CREATE VIEW cannot follow IF. This is the one that surprises people who know the first three do not exist. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT 1;\nCREATE VIEW dbo.probe_m_view2 AS SELECT 1 AS x;", + "expect": { + "error": 111 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And CREATE VIEW must be first in its batch, which is the same rule from the other side and the reason these objects need dynamic SQL or their own batch. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE OR ALTER TABLE dbo.probe_m_coa (id int);", + "expect": { + "error": 156 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. CREATE OR ALTER covers programmability objects and not tables, which is a distinction a model does not make. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_dupcol (id int);\nGO\nALTER TABLE dbo.probe_m_dupcol ADD id int;", + "expect": { + "error": 2705 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The number an unguarded migration produces on its second run, and the one the skill's eight concurrent runners hit seven times out of eight. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_dupobj (id int);\nGO\nCREATE TABLE dbo.probe_m_dupobj (id int);", + "expect": { + "error": 2714 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The object level version of the same non-idempotence. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_dupidx (id int);\nCREATE INDEX probe_m_dupidx_ix ON dbo.probe_m_dupidx (id);\nGO\nCREATE INDEX probe_m_dupidx_ix ON dbo.probe_m_dupidx (id);", + "expect": { + "error": 1913 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And the index level version. The skill lists these three together as the numbers that mean a script is not idempotent. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_col (id int);\nGO\nSELECT nosuchcolumn FROM dbo.probe_m_col;", + "expect": { + "error": 207 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. What an old connection sees after the shape changes underneath it, which is why the skill sequences a rename as add, backfill, switch, drop. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_m_res (id int, s nvarchar(50));\nGO\nALTER TABLE dbo.probe_m_res ALTER COLUMN s nvarchar(100) WITH (ONLINE = ON, RESUMABLE = ON);", + "expect": { + "error": 102 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. RESUMABLE is not available on ALTER COLUMN, only on index builds, so the answer a model gives for a long running column change does not parse. It goes red if the statement succeeds, or fails with any other number." + } + , + { + "sql": "CREATE TABLE dbo.probe_m_ver (version int NOT NULL PRIMARY KEY);\nGO\nINSERT INTO dbo.probe_m_ver VALUES (1);\nGO\nINSERT INTO dbo.probe_m_ver VALUES (1);", + "expect": { + "error": 2627 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The version row collision the skill's concurrency table records, and the one that arrives AFTER the backfill has already run eight times. It is the last error in that run and the least important one. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Is this migration safe to run against production while the app is live", + "Can I add this column to a big table without downtime", + "How do I do a zero downtime schema change on Azure SQL Database", + "Walk me through expand and contract for this column rename", + "Make this migration script idempotent so it is safe to re-run", + "Should my application run migrations at startup", + "Write the rollback for this migration", + "How do I do a blue green deploy when there is only one database", + "Will ALTER COLUMN int to bigint lock this table" + ], + "implicit": [ + "every query timed out for ten seconds during the deploy and then recovered", + "our deployment added one nullable column and the whole API went red", + "seven of our eight pods crashed on startup after we merged the migration", + "the migration says it applied but the backfilled values are twice what they should be", + "the migration ran twice and now the counts are wrong", + "we renamed a column and a stored procedure broke the next day", + "can I just drop the column and add it back to fix the type", + "the migration failed halfway and now it will not run again", + "our migration runner retried and reported success but the data looks wrong", + "the ALTER TABLE has been running for four minutes and everything is blocked" + ], + "routing": [ + "review this schema change before it ships", + "we need to change this column type on a live table" + ], + "negative": [ + "Set up a GitHub Actions workflow to deploy my database changes", + "Build a SQL Database Project and publish the DACPAC", + "Why does dotnet ef database update fail with a firewall error", + "Configure EnableRetryOnFailure on my DbContext", + "Prisma migrate dev needs a shadow database, how do I give it one", + "Write the Alembic env.py for my SQLAlchemy models", + "Should this key be an int or a uniqueidentifier", + "How long should the string columns in my index be", + "Deploy my container app to Azure", + "Why is this SELECT slow", + "Write me an upsert for this table" + ] + } +} diff --git a/plugins/microsoft-sql-fdh/skills/sql-database-projects b/plugins/microsoft-sql-fdh/skills/sql-database-projects deleted file mode 120000 index a816459..0000000 --- a/plugins/microsoft-sql-fdh/skills/sql-database-projects +++ /dev/null @@ -1 +0,0 @@ -../../../skills/sql-database-projects \ No newline at end of file diff --git a/plugins/microsoft-sql-fdh/skills/sql-database-projects/SKILL.md b/plugins/microsoft-sql-fdh/skills/sql-database-projects/SKILL.md new file mode 100644 index 0000000..186a63a --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/sql-database-projects/SKILL.md @@ -0,0 +1,306 @@ +--- +name: sql-database-projects +description: >- + Builds and publishes a SQL database project against Azure SQL Database: the SDK-style + `.sqlproj` on `Microsoft.Build.Sql`, the target platform that decides what the build actually + validates, pre and post deployment scripts, the refactorlog, and code analysis. Use when a user + asks to "create a SQL database project", "build a dacpac", "publish a dacpac to Azure SQL", + "add a post-deployment script", "rename a column without losing its data", or "turn on code + analysis", and reports "the build passed but the publish failed", "the deploy said success and + the data is gone", or "the post-deployment script failed and the table already changed". Covers + what `dotnet build` does not check, what SqlPackage does with a mismatched target platform, and + which publish options change data rather than schema. github-actions-for-sql owns the pipeline, + schema-migrations-safely the change doctrine. +--- + +# Build and publish a SQL database project + +**This owns the tooling: the project file, the build, the dacpac and the publish.** It does not +own how to sequence a schema change (`schema-migrations-safely`), running the publish from a +pipeline (`github-actions-for-sql`), or deploying the application beside it +(`deploy-app-to-azure`). Open `sqlpackage-import-export` when the job is moving a whole database +rather than building one, or when the artifact is a bacpac. + +Verified 2026-09-03 with the .NET SDK 8.0.421 on macOS, `Microsoft.Build.Sql` 2.2.0, +`Microsoft.Build.Sql.Templates` 2.2.0 and SqlPackage 170.4.83.3, publishing into the Azure SQL +Database container reporting `EngineEdition` 5, `SQL Azure`, `12.0.2000.8`. + +## The build runs anywhere, and SqlPackage is a separate install + +```bash +dotnet new install Microsoft.Build.Sql.Templates +dotnet new sqlproj -n ShopDb -o ShopDb --target-platform SqlAzureV12 +dotnet build ShopDb # writes ShopDb/bin/Debug/ShopDb.dacpac +dotnet tool install -g microsoft.sqlpackage +``` + +It is an SDK-style project, so `dotnet build` is the whole build story on Linux, macOS and Windows +alike, with no IDE and no Windows-only component. SqlPackage is not installed by the SDK, is not +on a hosted build image by default, and every publish below needs it. + +## The template's default target is not Azure SQL Database + +`dotnet new sqlproj --help` lists ten platforms and defaults to `Sql170`, the box product. The +choice lands in the project file as one property, and nothing in the build output announces it: + +```xml +Microsoft.Data.Tools.Schema.Sql.Sql170DatabaseSchemaProvider +Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider +``` + +A project holding `CREATE QUEUE [dbo].[OrderQueue];`, which Azure SQL Database has never +supported, built under both: + +| Target platform | `dotnet build` | Then `/Action:Publish` to Azure SQL Database | +|---|---|---| +| `Sql170`, the template default | **succeeds, 0 errors** | **refused before any change**, exit 1 | +| `SqlAzureV12` | **fails, 1 error** | never reached | + +Under `SqlAzureV12` the build names the file and the line: + +```text +OrderQueue.sql(1,1,1,1): Build error SQL70015: Statement 'CREATE QUEUE' is not supported for +the targeted platform. +``` + +Under the default the objection arrives from SqlPackage instead, during plan generation: + +```text +*** An error occurred during deployment plan generation. Deployment cannot continue. +A project which specifies as the target platform cannot be +published to Microsoft Azure SQL Database v12. +``` + +Nothing half-deploys from this cause: SqlPackage compares the dacpac's declared platform against +the server and stops. It still costs a release, because the build went green and the artifact was +tagged before anyone learned. Check an inherited project without editing it, by overriding the +provider on the command line: + +```bash +dotnet build ShopDb -t:Rebuild /p:DSP=Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider +``` + +If that fails where the ordinary build passed, the project is not targeting Azure SQL Database. +`/p:AllowIncompatiblePlatform=true` on the publish forces past the refusal instead of fixing it. +It defaults to `False`. Leave it there. + +## The right target validates statements, not references + +`SqlAzureV12` does not check that a referenced object exists. A procedure calling a mail or agent +stored procedure Azure SQL Database does not have builds under the Azure target with only: + +```text +Build warning SQL71502: Procedure: [dbo].[DoWork] has an unresolved reference to object +[dbo].[NoSuchTable]. +``` + +Warnings do not fail a build, so the procedure reaches the database and fails the first time it +runs. Answer each one, or escalate the whole class: + +```bash +dotnet build ShopDb -t:Rebuild -p:TreatTSqlWarningsAsErrors=true +``` + +Measured: every `SQL71502` becomes `Build error SQL71502` and the build fails. Where one warning +is genuinely fine, `SuppressTSqlWarnings` takes a comma-separated list of numbers. + +## Deployment scripts ride in the dacpac and are read by nothing + +The SDK globs `**/*.sql` into the model, then removes whatever is declared as a deployment script +or as `None`. There is a `RefactorLog` item type and no glob creates one. + +| Item type | Globbed | Parsed and validated | +|---|---|---| +| `Build`, ordinary object files | yes, `**/*.sql` | yes | +| `PreDeploy`, `PostDeploy` | no, declare them | **no, not at all** | +| `RefactorLog` | **no, declare it** | not applicable | + +```xml + + + + + +``` + +A post-deployment script whose entire body is `INSERT INTO dbo.NoSuchTable (id) VALUES (1);` +builds with **0 errors and 0 warnings**. The same statement inside a stored procedure produces +`SQL71502`. A project takes one pre-deployment and one post-deployment script; chain more with +`:r .\scripts\script1.sql` inside them, and add `` so the +chained files stay out of the model. Seed data belongs here, written so a second run is a no-op, +never in a `Build` file. + +## A failed publish does not roll back the schema + +Publishing a project that adds a column and carries a post-deployment script that fails: + +```text +Altering Table [dbo].[Product]... +Updating database (Failed) +Error SQL72014: ... Msg 208, Level 16, State 1, Line 2 Invalid object name 'dbo.NoSuchTable'. +Error SQL72045: Script execution error. +``` + +SqlPackage exits 1, **and the new column is there and stays there.** +`/p:IncludeTransactionalScripts=True` changes nothing here: deployment scripts run outside the +transactional block, which is exactly the part that failed. So a red pipeline step is not a +database that stayed still. Read the schema after a failed publish before deciding anything. +`schema-migrations-safely` owns why idempotent scripts are a rule rather than a nicety. + +## The rename, which is the one that costs data + +To a schema comparison a column rename is a drop and an add. The refactorlog turns it back into a +rename, and it is the easiest thing here to leave half-wired, because the file existing is not the +file being used. + + +Measured on a `Product` table holding two rows: + +| What the project has | Publish result | +|---|---| +| the rename, no refactorlog | blocked: `Msg 50000 ... Rows were detected. The schema update is terminating because data loss might occur.` exit 1 | +| the file on disk but **not declared** | **the same block.** The build succeeded and said nothing | +| declared as a `RefactorLog` item | `Rename [dbo].[Product].[Name] to ProductName`, both rows keep their values, exit 0 | +| no refactorlog, `/p:BlockOnPossibleDataLoss=False` | `Successfully published database`, **exit 0, both values now null** | + +The last row is the whole reason this skill exists: turning that option off to get a pipeline +green converts a blocked deployment into a successful one that dropped a column. + +You do not need a database to find out which row you are in. DeployReport reads a dacpac against +another dacpac, so the plan is readable before any server is involved: + +```bash +sqlpackage /Action:DeployReport /SourceFile:After/bin/Debug/After.dacpac \ + /TargetFile:Before/bin/Debug/Before.dacpac /TargetDatabaseName:ShopDb /OutputPath:plan.xml +``` + +Measured 2026-09-03, the same rename twice. Without the refactorlog declared: + +```xml + +``` + +With it declared, the alert is gone and the operation changed: + +```xml + +``` + +The first publish carrying a refactorlog creates `dbo.__RefactorLog` in the target and records the +operation key, so each rename applies once. That table is deployment state: copying a database +without it makes an already-applied rename look outstanding. + +## The database is not the project + +`/p:DropObjectsNotInSource` defaults to **False**. Measured: a table and a procedure created by +hand in the target both survive a publish of a project that contains neither, and the publish +reports success. Turning it on drops them and keeps `__RefactorLog`: + +```text +Dropping Table [dbo].[Legacy]... +Dropping Procedure [dbo].[LegacyProc]... +``` + +Neither setting is the safe one. Decide whether this database is a mirror of the project or a +superset holding things the project does not describe, and never turn the option on without +reading a plan first. + +## Code analysis is on request, and advisory until escalated + +```xml +true +``` + +Measured on a procedure using `SELECT *` and `@@IDENTITY`: `StaticCodeAnalysis warning SR0001`, +`SR0008`, and `Build succeeded. 0 Error(s), 2 Warning(s)`. Warnings in a log nobody reads are not +a gate. Escalate one rule with `+!`, which fails the build: + +```bash +dotnet build ShopDb -t:Rebuild -p:SqlCodeAnalysisRules='+!Microsoft.Rules.Data.SR0001' +``` + +Measured: `StaticCodeAnalysis error SR0001`, 1 error and 1 warning, `Build FAILED`. Two traps. The +`sqlcodeanalysis` template does **not** turn analysis on: it scaffolds a C# project for writing a +custom rule, which then has to be packaged and referenced. And `dotnet build` skips analysis +entirely on a project that is up to date, so a rerun reporting zero warnings may have analysed +nothing. Use `-t:Rebuild` whenever the warning count is the answer you want. + +## Publishing + +```bash +sqlpackage /Action:Publish /SourceFile:ShopDb/bin/Debug/ShopDb.dacpac \ + /TargetConnectionString:"Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=True;" +``` + +Read the plan against the real target before applying it. Both of these write a file and change +nothing: + +```bash +sqlpackage /Action:DeployReport /SourceFile:ShopDb/bin/Debug/ShopDb.dacpac \ + /TargetConnectionString:"Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=True;" \ + /OutputPath:plan.xml + +sqlpackage /Action:Script /SourceFile:ShopDb/bin/Debug/ShopDb.dacpac \ + /TargetConnectionString:"Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=True;" \ + /DeployScriptPath:publish.sql +``` + +`/OutputPath` is the DeployReport parameter and DeployReport does not accept `/DeployReportPath`; +`github-actions-for-sql` owns that trap and the pipeline around it. + +`/Action:Publish` does not create the logical server, and in Azure that has to exist first. +**It does create the database.** Measured against the container reporting `EngineEdition` 5: +publishing a dacpac at a database name that did not exist printed `Creating database ...`, +then created the table, and the row count in `sys.databases` went from 0 to 1. Do not add a create +step before a publish on the assumption that it is required. + +## Check it worked + +Three checks in the order the failures happen. The first two need no database at all. + +The dacpac declares the platform the publish will compare against the server, and that is the +value that decides the refusal, not the text in the project file: + +```bash +unzip -p ShopDb/bin/Debug/ShopDb.dacpac model.xml | grep -o 'DspName="[^"]*"' +``` + +Expect exactly `DspName="Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider"`. +Anything else and the publish to Azure SQL Database is refused during plan generation. + +Everything the build refuses to read sits in the package under a fixed name: + +```bash +unzip -l ShopDb/bin/Debug/ShopDb.dacpac +``` + +Expect `refactor.xml` if this release carries a rename, `postdeploy.sql` if it carries a +post-deployment script, and `predeploy.sql` if it carries a pre-deployment one. Measured +2026-09-03: undeclared, the dacpac holds four entries and no `refactor.xml`; declared, five. A +missing entry means the item was never declared, and the build did not complain. + +Then, after the publish, ask the target whether anything is left: + +```bash +sqlpackage /Action:DeployReport /SourceFile:ShopDb/bin/Debug/ShopDb.dacpac \ + /TargetConnectionString:"" /OutputPath:after.xml +grep -c "` with nothing inside it. A publish that landed leaves the next +deploy report nothing to do, so any count above zero names an object the run did not apply. + +## References + +- Open [target-platform-and-refactorlog-runs.md](references/target-platform-and-refactorlog-runs.md) when a + table above needs reproducing, when a build or publish message you are seeing does not match one + quoted here, or to look up a property or SqlPackage option this body only names in passing. +- `sdk/Sdk.props` and `sdk/Sdk.targets` inside the `Microsoft.Build.Sql` package in the local + NuGet cache: read them when a file is sitting in the project directory and is not in the build. +- Microsoft Learn is the authority on whether a property or item type exists. Open + [SQL projects properties](https://learn.microsoft.com/sql/tools/sql-database-projects/concepts/project-properties), + [target platform](https://learn.microsoft.com/sql/tools/sql-database-projects/concepts/target-platform) + or [refactoring](https://learn.microsoft.com/sql/tools/sql-database-projects/concepts/refactor-overview) + before writing a property this body does not name. diff --git a/plugins/microsoft-sql-fdh/skills/sql-database-projects/references/target-platform-and-refactorlog-runs.md b/plugins/microsoft-sql-fdh/skills/sql-database-projects/references/target-platform-and-refactorlog-runs.md new file mode 100644 index 0000000..000bd1d --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/sql-database-projects/references/target-platform-and-refactorlog-runs.md @@ -0,0 +1,185 @@ +# Measured behaviour of the build and the publish + +## Contents + +- [How these were run](#how-these-were-run) +- [Target platform against the build](#target-platform-against-the-build) +- [Target platform against the publish](#target-platform-against-the-publish) +- [What the build parses](#what-the-build-parses) +- [Atomicity of a publish](#atomicity-of-a-publish) +- [Rename, refactorlog and data loss](#rename-refactorlog-and-data-loss) +- [Objects in the target and not in the project](#objects-in-the-target-and-not-in-the-project) +- [Code analysis](#code-analysis) +- [Project file properties worth setting](#project-file-properties-worth-setting) +- [SqlPackage options this skill relies on](#sqlpackage-options-this-skill-relies-on) + +## How these were run + +Every result below came from building a project and publishing it, on 2026-08-28, with: + +| Component | Version | +|---|---| +| .NET SDK | 8.0.421, on macOS on an arm64 host | +| `Microsoft.Build.Sql` | 2.2.0, the current stable release | +| `Microsoft.Build.Sql.Templates` | 2.2.0 | +| SqlPackage | 170.4.83.3, installed with `dotnet tool install -g microsoft.sqlpackage` | +| Target engine | the Azure SQL Database container, `EngineEdition` 5, `SQL Azure`, `12.0.2000.8` | + +The project is four files: a table, a Service Broker queue, a table using a filegroup clause, and +a procedure calling a mail stored procedure. + +## Target platform against the build + +`dotnet new sqlproj` offers ten target platforms and defaults to the newest box-product one. The +choice is written into the project as a single `DSP` property, and the schema provider for Azure +SQL Database is `Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider`. + +| Source file | Default target | `SqlAzureV12` | +|---|---|---| +| `CREATE QUEUE [dbo].[OrderQueue];` | accepted, no diagnostic | `Build error SQL70015: Statement 'CREATE QUEUE' is not supported for the targeted platform.` | +| `CREATE TABLE ... ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]` | accepted | accepted, no diagnostic either | +| procedure calling `sys.sp_send_dbmail` | `Build warning SQL71502`, four of them | the same four warnings, still warnings | + +Two things follow. The target platform catches statements and not references, and it does not +catch everything a reader might expect it to: a filegroup clause passes under both. + +Escalating the warning class turns the third row into a build failure: + +```bash +dotnet build -t:Rebuild -p:TreatTSqlWarningsAsErrors=true +# 4 Error(s), Build FAILED. +``` + +## Target platform against the publish + +Publishing a dacpac whose declared platform is not Azure SQL Database: + +```text +Publishing to database 'shopdb' on server ''. +Initializing deployment (Start) +Initializing deployment (Failed) +*** An error occurred during deployment plan generation. Deployment cannot continue. +A project which specifies as the target platform cannot be published to +Microsoft Azure SQL Database v12. +Time elapsed 0:00:01.45 +``` + +Exit code 1. The message names the platform the project declared. Three different non-Azure +targets were tried and all three were refused with the same shape of message, at the same point, +before the deployment plan existed. Nothing is written to the database. + +## What the build parses + +`Sdk.props` globs `Build` items as `**/*.sql`, and `Sdk.targets` then removes from that glob +anything declared as `PreDeploy`, `PostDeploy`, `None` or an extension configuration. There is a +`RefactorLog` item type, and **no glob creates one**. + +Evidence for the deployment scripts being unparsed: a post-deployment script whose entire body is +`INSERT INTO dbo.NoSuchTable (id) VALUES (1);` builds with `0 Error(s), 0 Warning(s)`. The same +statement inside a procedure produces `SQL71502`. + +Evidence for the refactorlog not being globbed: a file named `.refactorlog` sitting +beside the project produced a dacpac containing four entries and no `refactor.xml`. Adding +`` produced five entries including `refactor.xml`. + +## Atomicity of a publish + +One publish that adds a column and then runs a failing post-deployment script: + +```text +Updating database (Start) +Altering Table [dbo].[Product]... +post-deploy starting +An error occurred while the batch was being executed. +Updating database (Failed) +*** Could not deploy package. +Error SQL72014: Core Microsoft SqlClient Data Provider: Msg 208, Level 16, State 1, Line 2 + Invalid object name 'dbo.NoSuchTable'. +Error SQL72045: Script execution error. +``` + +Exit code 1. `sys.columns` afterwards still lists the added column. + +The run was repeated with `/p:IncludeTransactionalScripts=True` after dropping the column again. +Same failure, and the column is present afterwards again. The option does not put a deployment +script inside the transaction. + +## Rename, refactorlog and data loss + +`dbo.Product` held two rows. The project renamed `Name` to `ProductName`. + +| Configuration | Output | Exit | Data | +|---|---|---|---| +| no refactorlog | `Warning SQL72015: The column [dbo].[Product].[Name] is being dropped, data loss could occur.` then `Msg 50000 ... Rows were detected. The schema update is terminating because data loss might occur.` | 1 | unchanged | +| refactorlog present but not declared | identical to the row above | 1 | unchanged | +| refactorlog declared | `The following operation was generated from a refactoring log file ` / `Rename [dbo].[Product].[Name] to ProductName` | 0 | both values preserved | +| no refactorlog, `/p:BlockOnPossibleDataLoss=False` | `*** The column [dbo].[Product].[Name] is being dropped, data loss could occur.` then `Starting rebuilding table [dbo].[Product]...` then `Successfully published database.` | **0** | **both values null** | + +The last row printed eleven lines in total. The only warning sat above `Initializing deployment +(Complete)`, and the final line was a success. + +When a rename is applied, `dbo.__RefactorLog` appears in the target database and records the +operation key, so the same rename is not attempted twice. + +## Objects in the target and not in the project + +A table and a procedure were created directly in the target. The project contained neither. + +| Publish | Result | +|---|---| +| default options | `Successfully published database.` Both objects still present | +| `/p:DropObjectsNotInSource=True` | `Dropping Table [dbo].[Legacy]...` / `Dropping Procedure [dbo].[LegacyProc]...` Both gone, `__RefactorLog` kept | + +## Code analysis + +`true` in the project runs the rule set during +`dotnet build`. On a procedure using `SELECT *` and `@@IDENTITY`: + +```text +StaticCodeAnalysis warning SR0001: Microsoft.Rules.Data : The shape of the result set produced + by a SELECT * statement will change if the underlying table or view structure changes. +StaticCodeAnalysis warning SR0008: Microsoft.Rules.Data : Potential misuse of system function + @@IDENTITY. +Build succeeded. 0 Error(s) 2 Warning(s) +``` + +Escalating one rule to an error, which fails the build: + +```bash +dotnet build -t:Rebuild -p:SqlCodeAnalysisRules='+!Microsoft.Rules.Data.SR0001' +# StaticCodeAnalysis error SR0001: ... +# 1 Error(s), Build FAILED. +``` + +Without `-t:Rebuild` an up-to-date project reports `0 Warning(s)` because it does not rebuild the +model, so the analysis never runs. A run that reports nothing may have analysed nothing. + +The `sqlcodeanalysis` template creates a C# project targeting `netstandard2.1` with a +`Microsoft.SqlServer.DacFx` package reference and a sample rule class. It is for authoring a +custom rule and packaging it, not for enabling analysis. + +## Project file properties worth setting + +| Property or item | Why | +|---|---| +| `` | the target platform. The only thing deciding which rules the build applies | +| `` | runs the rule set inside the build | +| `` | turns unresolved references and their kind into build failures | +| `` | `+!` escalates one rule to an error, `-` disables one | +| `` | the one script that runs after the schema changes. Unparsed, so make it idempotent | +| `` | runs before them, and is equally unparsed | +| `` | never globbed. Without this line the rename is a drop and an add | + +## SqlPackage options this skill relies on + +| Option | Default | Effect | +|---|---|---| +| `/p:BlockOnPossibleDataLoss` | `True` | stops a deployment that would drop data. Turning it off turns the block into a silent success | +| `/p:DropObjectsNotInSource` | `False` | the target keeps objects the project does not describe | +| `/p:IncludeTransactionalScripts` | `False` | wraps generated schema statements, and not the deployment scripts | +| `/Action:DeployReport` | | writes the planned changes as XML and changes nothing | +| `/Action:Script` | | writes the deployment T-SQL and changes nothing | +| `/Action:DriftReport` | | reports how the target has diverged from what was last published | + +`Extract`, `Import` and `Export` are SqlPackage actions but are not accepted by the GitHub action +wrapper, which is covered in `github-actions-for-sql`. diff --git a/plugins/microsoft-sql-fdh/skills/sql-database-projects/skill.spec.jsonc b/plugins/microsoft-sql-fdh/skills/sql-database-projects/skill.spec.jsonc new file mode 100644 index 0000000..bd352a4 --- /dev/null +++ b/plugins/microsoft-sql-fdh/skills/sql-database-projects/skill.spec.jsonc @@ -0,0 +1,158 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "sql-database-projects", + "domain": "schema-change-and-cicd", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked for a SQL database project for Azure SQL Database, an agent runs `dotnet new sqlproj`, accepts the template's default target platform, and reads a green `dotnet build` as proof the schema is valid for Azure SQL Database. It is not. With the default target the build accepts statements Azure SQL Database has never supported and reports zero errors, and the refusal arrives only at publish, where SqlPackage stops during deployment plan generation and exits 1, so the cost is a release that fails after the artifact was built and handed on. Setting the target platform to SqlAzureV12 turns that same statement into build error SQL70015, which is necessary and is then over-read: the correct target platform does not validate a call to a server feature Azure SQL Database does not have, which stays warning SQL71502 and reaches the database intact, and it does not validate pre and post deployment scripts at all, which no parser reads and which run outside the deployment transaction even when IncludeTransactionalScripts is set, so a script that fails leaves the schema changes already applied and committed. The expensive one is silent and lands after a successful deploy: the SDK never globs a `.refactorlog` file, so unless it is declared as a RefactorLog item the build succeeds without it and the publish plans a column drop and add instead of a rename, which BlockOnPossibleDataLoss blocks until somebody sets it to False to get the pipeline green, at which point the publish prints `Successfully published database`, exits 0, and the column's data is gone.", + "value_probe": { + "gate": "The build itself is made to refuse what the target database will refuse, and a rename is carried explicitly by the project rather than inferred at deployment time from the changed schema.", + "task": "Our schema lives in a database project in the repository and CI builds it on every pull request. Two things happened last month that I need to stop happening again. A build that was green failed at deployment time against our Azure database. And a change that was meant to rename a column ended with the new column empty in production, while the deployment reported success and exited zero. Both went through review and nobody caught either one. What should I change about the project and about the pipeline?", + "states": [ + "The project's target platform has to name the cloud service, or statements that service rejects will build without error.", + "The record of a rename has to be declared as an item in the project, or the build succeeds without it and the deployment plans a drop and an add.", + "A correct target platform still does not validate calls to server features that are absent here, which stay warnings and reach the database.", + "Pre and post deployment scripts are not parsed by the build and run outside the deployment transaction." + ], + "refutes": [ + "A green build of the database project means the schema is valid for Azure SQL Database.", + "Renaming a column in the project produces a rename at deployment time, so the data in it is not at risk.", + "Turning off the block on possible data loss is a reasonable way to get a blocked deployment through.", + "A deployment that prints success and exits zero has preserved the data in the columns it changed." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "schema-migrations-safely", + "github-actions-for-sql", + "deploy-app-to-azure", + "dev-container-templates", + "design-azure-sql-schema", + "provision-azure-sql-db", + "ef-core-azure-sql", + "azuresql-db-import", + "azuresql-db-schema-migration" + ], + "validation": { + "target": "both", + "setup": [ + "the .NET SDK, so `dotnet new sqlproj` and `dotnet build` are available on any host operating system", + "SqlPackage installed as a global .NET tool, because nothing else installs it", + "a reachable server, since publishing does not create the logical server; the target database itself does not need to exist, because publish creates it", + "a table carrying rows, so the data-loss and rename behaviour is observable rather than theoretical" + ], + "assert": [ + "the project's target platform is read out of the .sqlproj DSP property and reported, rather than assumed from the template default", + "the target platform is the Azure SQL Database schema provider whenever the deployment target is Azure SQL Database, and a build that passes under any other target is not treated as evidence about Azure SQL Database", + "a build reporting zero errors is not reported as a validated schema until the publish has generated a deployment plan against the real target", + "unresolved-reference warnings are enumerated rather than ignored, because the correct target platform does not turn a call to an absent server feature into an error", + "every pre and post deployment script is stated to be unvalidated by the build, and the plan says what a half-applied deployment would leave behind", + "a column rename carries a refactorlog entry AND the refactorlog is declared as a RefactorLog item in the project, verified by the presence of refactor.xml inside the built dacpac", + "BlockOnPossibleDataLoss is left on, and any request to turn it off is answered with what would be dropped", + "whether objects present in the database and absent from the project are meant to be dropped is decided explicitly, because the default leaves them in place", + "code analysis results are read as warnings that do not fail a build until a rule or the whole set is escalated" + ], + "probes": [ + { + "where": "host", + "cmd": "dotnet new sqlproj --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'SqlAzureV12' && dotnet new sqlproj --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'Default: Sql170'", + "expect": { + "succeeds": true + }, + "why": "The premise of this skill's first correction: the template offers the Azure SQL Database platform and does not choose it. Measured 2026-09-04 on Microsoft.Build.Sql.Templates 2.2.0, where --target-platform lists ten values and prints 'Default: Sql170'. Both halves are asserted together because either one moving invalidates the section: if SqlAzureV12 disappears the skill's instruction cannot be followed, and if the default ever becomes SqlAzureV12 the whole build-green-publish-red story stops being true and the skill should say so rather than keep warning about it." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new sqlproj -n QueueDb -o QueueDb >/dev/null 2>&1 && printf 'CREATE QUEUE [dbo].[OrderQueue];\\n' > QueueDb/OrderQueue.sql && dotnet build QueueDb -v q --nologo >/dev/null 2>&1 && unzip -p QueueDb/bin/Debug/QueueDb.dacpac model.xml | grep -q 'DspName=\"Microsoft.Data.Tools.Schema.Sql.Sql170DatabaseSchemaProvider\"'", + "expect": { + "succeeds": true + }, + "why": "The top row of the target-platform table, and the reason the failure is expensive: CREATE QUEUE is a statement Azure SQL Database has never supported, and under the template default the build accepts it, reports zero errors and ships a dacpac. The probe then reads the platform out of the built package rather than out of the project file, because the value inside model.xml is the one SqlPackage compares against the server. Measured 2026-09-04 on .NET SDK 8.0.421. It goes red if the build ever rejects the statement under the default target, which would mean the artifact can no longer be built green and handed on, or if the package stops declaring its provider where the skill's check reads it." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new sqlproj -n QueueDb -o QueueDb -tp SqlAzureV12 >/dev/null 2>&1 && printf 'CREATE QUEUE [dbo].[OrderQueue];\\n' > QueueDb/OrderQueue.sql && dotnet build QueueDb -v q --nologo 2>&1 | grep -q \"Build error SQL70015: Statement 'CREATE QUEUE' is not supported for the targeted platform.\"", + "expect": { + "succeeds": true + }, + "why": "The second row of the same table, and it has to be a separate probe because the claim is a difference between two builds of identical source. Under SqlAzureV12 the same file is a build error naming the statement, SQL70015, measured 2026-09-04 with the full message text asserted rather than the code alone so a renumbering and a rewording are both visible. If this ever passes the build, the target platform stops being the thing that catches an unsupported statement and the skill's central instruction has no effect." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new sqlproj -n RefDb -o RefDb -tp SqlAzureV12 >/dev/null 2>&1 && printf 'CREATE PROCEDURE [dbo].[DoWork] AS SELECT 1 FROM dbo.NoSuchTable;\\n' > RefDb/DoWork.sql && dotnet build RefDb -v q --nologo 2>&1 | grep -q 'Build warning SQL71502' && dotnet build RefDb -t:Rebuild -v q --nologo >/dev/null 2>&1 && ! dotnet build RefDb -t:Rebuild -v q --nologo -p:TreatTSqlWarningsAsErrors=true >/dev/null 2>&1", + "expect": { + "succeeds": true + }, + "why": "The over-reading this skill exists to stop: setting the correct target platform does not make an unresolved reference an error. The same source builds green with SQL71502 as a warning and fails once TreatTSqlWarningsAsErrors is set, measured 2026-09-04. Both halves are in one probe because the claim is the contrast, and a probe asserting only the warning would stay green on a build where warnings had silently become fatal. The middle rebuild is deliberate: the skill records that dotnet build skips work on an up-to-date project, so each half is forced to actually run." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new sqlproj -n PostOnly -o PostOnly -tp SqlAzureV12 >/dev/null 2>&1 && printf 'INSERT INTO dbo.NoSuchTable (id) VALUES (1);\\n' > PostOnly/Script.PostDeployment.sql && sed -i.bak 's###' PostOnly/PostOnly.sqlproj && dotnet build PostOnly -v q --nologo 2>&1 | tr -s '[:space:]' ' ' | grep -q '0 Warning(s) 0 Error(s)' && unzip -l PostOnly/bin/Debug/PostOnly.dacpac | grep -q 'postdeploy.sql'", + "expect": { + "succeeds": true + }, + "why": "Deployment scripts ride in the package and are read by nothing. The identical statement that produces SQL71502 inside a stored procedure produces zero warnings and zero errors in a post-deployment script, and the file is still carried into the dacpac as postdeploy.sql, measured 2026-09-04. That gap is why a publish can fail after the schema change has already committed. The probe asserts both the silence and the presence, because silence alone would also be the result of the script having been dropped from the build entirely, which is a different and equally important failure the skill warns about." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new sqlproj -n Before -o Before -tp SqlAzureV12 >/dev/null 2>&1 && dotnet new sqlproj -n After -o After -tp SqlAzureV12 >/dev/null 2>&1 && printf 'CREATE TABLE [dbo].[Product] ([Id] INT NOT NULL PRIMARY KEY, [Name] NVARCHAR(50) NULL);\\n' > Before/Product.sql && printf 'CREATE TABLE [dbo].[Product] ([Id] INT NOT NULL PRIMARY KEY, [ProductName] NVARCHAR(50) NULL);\\n' > After/Product.sql && printf '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n' > After/After.refactorlog && dotnet build Before -v q --nologo >/dev/null 2>&1 && dotnet build After -v q --nologo >/dev/null 2>&1 && ! unzip -l After/bin/Debug/After.dacpac | grep -q refactor.xml && sqlpackage /Action:DeployReport /SourceFile:After/bin/Debug/After.dacpac /TargetFile:Before/bin/Debug/Before.dacpac /TargetDatabaseName:ShopDb /OutputPath:undeclared.xml >/dev/null 2>&1 && grep -q '##' After/After.sqlproj && dotnet build After -t:Rebuild -v q --nologo >/dev/null 2>&1 && unzip -l After/bin/Debug/After.dacpac | grep -q refactor.xml && sqlpackage /Action:DeployReport /SourceFile:After/bin/Debug/After.dacpac /TargetFile:Before/bin/Debug/Before.dacpac /TargetDatabaseName:ShopDb /OutputPath:declared.xml >/dev/null 2>&1 && grep -q '' declared.xml && grep -q '/dev/null 2>&1 || { echo \"template failed\"; exit 1; }\nprintf 'CREATE TABLE [dbo].[Keep] ([Id] INT NOT NULL PRIMARY KEY, [Payload] NVARCHAR(50) NOT NULL);\\n' > LossDb/Keep.sql\nprintf 'IF NOT EXISTS (SELECT 1 FROM [dbo].[Keep]) INSERT INTO [dbo].[Keep] ([Id], [Payload]) VALUES (1, N'\"'\"'irreplaceable'\"'\"');\\n' > LossDb/Script.PostDeployment.sql\nsed -i.bak 's###' LossDb/LossDb.sqlproj\ndotnet build LossDb -v q --nologo >/dev/null 2>&1 || { echo \"build 1 failed\"; exit 1; }\nsqlpackage /Action:Publish /SourceFile:LossDb/bin/Debug/LossDb.dacpac /TargetServerName:\"$srv\" /TargetDatabaseName:\"$PROBE_DB\" /TargetUser:sa /TargetPassword:\"$pw\" /TargetTrustServerCertificate:True >log1 2>&1 || { echo \"publish 1 failed\"; tail -5 log1; exit 1; }\nprintf 'CREATE TABLE [dbo].[Keep] ([Id] INT NOT NULL PRIMARY KEY);\\n' > LossDb/Keep.sql\nrm -f LossDb/Script.PostDeployment.sql\ndotnet new sqlproj -n LossDb2 -o LossDb2 -tp SqlAzureV12 >/dev/null 2>&1\nprintf 'CREATE TABLE [dbo].[Keep] ([Id] INT NOT NULL PRIMARY KEY);\\n' > LossDb2/Keep.sql\ndotnet build LossDb2 -v q --nologo >/dev/null 2>&1 || { echo \"build 2 failed\"; exit 1; }\nsqlpackage /Action:Publish /SourceFile:LossDb2/bin/Debug/LossDb2.dacpac /TargetServerName:\"$srv\" /TargetDatabaseName:\"$PROBE_DB\" /TargetUser:sa /TargetPassword:\"$pw\" /TargetTrustServerCertificate:True >log2 2>&1 && { echo \"the drop was published with data in the column\"; exit 1; }\ngrep -qi \"data loss\" log2 || { echo \"it failed, but not for data loss:\"; tail -5 log2; exit 1; }\nsqlpackage /Action:Publish /SourceFile:LossDb2/bin/Debug/LossDb2.dacpac /TargetServerName:\"$srv\" /TargetDatabaseName:\"$PROBE_DB\" /TargetUser:sa /TargetPassword:\"$pw\" /TargetTrustServerCertificate:True /p:BlockOnPossibleDataLoss=False >log3 2>&1 || { echo \"the override did not publish:\"; tail -5 log3; exit 1; }\ngrep -q \"Successfully published database\" log3 || { echo \"no success line under the override\"; tail -5 log3; exit 1; }\nexit 0", + "expect": { + "succeeds": true + }, + "why": "The claim that costs data, run end to end against a live engine rather than read off a help screen. The probe above asserts that BlockOnPossibleDataLoss defaults to True; this one asserts what that default does. A dacpac carrying a column and a post-deployment script that seeds a row is published into the lane's scratch database, then a second dacpac built from a model with that column removed is published at the defaults: SqlPackage refuses it naming possible data loss and exits non-zero. The same dacpac published with /p:BlockOnPossibleDataLoss=False prints 'Successfully published database' and exits 0, having dropped the column and the row in it. Measured 2026-09-05 against the container with SqlPackage 170.4.83.3. It has three separate ways to go red, the first publish failing, the blocked publish succeeding and the override publish failing, so a change in either the deployment plan or the engine's ALTER behaviour is caught rather than absorbed. It replaces an earlier version that published to a database that did not exist: the container answers CREATE DATABASE with Msg 42019, a timeout it reports while doing the work anyway, and SqlPackage reads that as a failed deployment, so the publish-creates-the-database claim is not settleable in this lane." + } + ] + }, + "triggering": { + "explicit": [ + "Use the sql-database-projects skill", + "create a SQL database project for my Azure SQL database", + "build this sqlproj and publish the dacpac to Azure SQL Database", + "add a post-deployment script to my database project", + "turn on code analysis for this SQL project" + ], + "implicit": [ + "the project builds fine but the dacpac will not deploy", + "why does my dacpac say it cannot be published to Microsoft Azure SQL Database v12", + "rename a column in my database project without losing the data in it", + "the deployment reported success and the column values are all null now", + "the publish failed halfway and the table already has the new column", + "how do I stop the publish dropping tables that are not in my project" + ], + "routing": [ + "what target platform should this database project use", + "where do I put seed data in a database project", + "why is my post-deployment script not being checked at build time", + "generate the T-SQL script instead of deploying it", + "what is the refactorlog for" + ], + "negative": [ + "write a GitHub Actions workflow that deploys my database project on merge", + "set up federated credentials so the pipeline logs in to Azure without a password", + "should I use expand and contract for this column change", + "add an Entity Framework Core migration for this new column", + "deploy my web app and its database to Azure with azd", + "set this project up from the Azure SQL dev container template", + "create an Azure SQL database and open the firewall for my address", + "grant my application identity a database user and roles" + ] + } +} diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server deleted file mode 120000 index d1be456..0000000 --- a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server +++ /dev/null @@ -1 +0,0 @@ -../../../skills/azuresql-db-from-sql-server \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/SKILL.md b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/SKILL.md new file mode 100755 index 0000000..b4a570c --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/SKILL.md @@ -0,0 +1,220 @@ +--- +name: azuresql-db-from-sql-server +description: >- + Migrates a local SQL Server setup to the Azure SQL Database container for Azure-faithful + local development. Use when a project already uses + mcr.microsoft.com/mssql/server, mssql/server, an sqlcmd plus SA password + docker setup, a "SQL Server in docker" or "local mssql container", or a + docker-compose with the mssql/server image; and use when the user asks for + "SQL Server locally", "run mssql in Docker", "spin up a local SQL database", + or "test against SQL Server" but actually wants the Azure SQL Database engine + (EngineEdition 5). Detects the SQL Server image, rewrites it to the Azure + SQL Database container, adds --platform on non-x64 hosts, keeps the SA login, flags SQL + Server-only features (SQL Agent, FILESTREAM, full Service Broker, + cross-server distributed transactions, Windows Auth), and re-points + connection strings from master to a provisioned user database. +--- + +# Migrate from the SQL Server image to the Azure SQL Database container + +This skill converts an existing local SQL Server setup (the SQL Server image +`mcr.microsoft.com/mssql/server`) into the **Azure SQL Database** container so +local dev matches Azure SQL Database behavior. The two are not the same engine: + +| | SQL Server image | Azure SQL Database container | +|---|---|---| +| Image | `mcr.microsoft.com/mssql/server` | `sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest` | +| `SERVERPROPERTY('EngineEdition')` | 2/3/4/8 | **5** | +| `SERVERPROPERTY('Edition')` | e.g. 'Developer Edition' | **'SQL Azure'** | +| DB model | one instance, many DBs, `USE` works | master for provisioning only; user DB for work; a user-database session returns `Msg 40508` on `USE`, exactly as in the cloud; a `master` connection is a provisioning session where the filter is not enforced | + +If a project is using the SQL Server image but wants Azure-faithful local dev, **stop +and switch to the container.** This skill is self-contained; +for full container detail see the **azuresql-db-container** skill. + +Verified on 2026-09-05 against the container image +`sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest`, reporting `EngineEdition` +5, Edition `SQL Azure`, build `12.0.2000.8`. All nine executable checks behind this skill +passed, including `Msg 40508` for `USE`, `Msg 40510` for `BACKUP`, `Msg 2812` for +`sp_configure` (absent rather than refused), `Msg 40517` for `ALTER DATABASE ... SET +RECOVERY`, the single-database model, `/docker-entrypoint-initdb.d` not being auto-run, and +`sqlcmd` at `/opt/mssql-tools18/bin/sqlcmd`. + +## When to use + +- The project Dockerfile, compose file, or run script references + `mcr.microsoft.com/mssql/server` or `mssql/server`. +- There is an sqlcmd plus SA password setup, or someone says "SQL Server in + Docker" / "a local mssql container". +- The user asks for "SQL Server locally" but actually targets Azure SQL Database. + +## Migration steps + +### 1. Detect SQL Server usage + +Search the repo for the SQL Server image and the patterns that move with it: + +```bash +grep -rniE 'mcr\.microsoft\.com/mssql/server|mssql/server|ACCEPT_EULA|MSSQL_SA_PASSWORD|SA_PASSWORD|sqlcmd|Server=localhost,1433|Database=master' . 2>/dev/null +``` + +Anything that hits is a candidate for rewrite below. + +### 2. Sign in to the preview registry + +The image is in a private preview registry; sign in once with the shared +pull-only credentials provided when you sign up at https://aka.ms/sqldbcontainerpreview-signup (they may +be rotated during the preview): + +```bash +docker login sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io +``` + +Registry and tag are provisional during Private Preview. + +### 3. Rewrite the image and add --platform + +Replace the SQL Server image with the Azure SQL Database image. The Azure image is x64 +only, so on a non-x64 host add `--platform linux/amd64` +(Docker) or `platform: linux/amd64` (compose). + +- Old: `mcr.microsoft.com/mssql/server:2022-latest` +- New: `sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest` + +Keep `ACCEPT_EULA=Y`, keep the complex `MSSQL_SA_PASSWORD`, keep the SA login, +keep port 1433. See [references/migrate-compose.md](references/migrate-compose.md) for a before/after compose. + +### 4. Start the container and provision appdb first + +The engine does **not** auto-create databases on connect, and it is **not** +ready the instant `docker run` returns. Use the canonical start recipe: it picks +a free host port, adds `--platform` only on non-x64 hosts, waits for readiness +with a retry loop, and provisions `appdb` inside that same loop. The `-b -l 2` +flags make a SQL error set the exit code so transient startup errors (e.g. +`Msg 913`) are retried, not masked. + +```bash +HOST_PORT=1433; while lsof -nP -iTCP:"$HOST_PORT" -sTCP:LISTEN >/dev/null 2>&1; do HOST_PORT=$((HOST_PORT+1)); done +PLATFORM=(); case "$(docker info -f '{{.Architecture}}' 2>/dev/null)" in x86_64|amd64) ;; *) PLATFORM=(--platform linux/amd64);; esac +docker rm -f sqldb 2>/dev/null +docker run -d --name sqldb "${PLATFORM[@]}" -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=YourStr0ng_Passw0rd" \ + -p "$HOST_PORT:1433" sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest +until docker exec sqldb /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "YourStr0ng_Passw0rd" -C -b -l 2 \ + -Q "IF DB_ID('appdb') IS NULL CREATE DATABASE appdb;" >/dev/null 2>&1; do sleep 2; done +echo "ready on localhost,$HOST_PORT" +``` + +A `master` connection is for provisioning only; do real work on `appdb`. + +### 5. Re-point connection strings from master to the user DB + +SQL Server projects often connect straight to `master` (or rely on a DB that SQL Server auto-creates). Azure SQL Database will not auto-create a database. Select the +target database in the connection string (`Database=appdb`, or `-d appdb` for +sqlcmd). Avoid `USE` to switch databases. In a user-database session (the +Azure-faithful context where you develop), `USE` returns `Msg 40508`, exactly as +in Azure SQL Database in the cloud. A `master` connection is a provisioning +session where the Azure statement filter is not enforced, so `USE` appears to work there, but `master` is for provisioning +only, not application work. Standardize on one `SQL_CONNECTION_STRING` env var: + +``` +Server=localhost,1433;Database=appdb;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true +``` + +House style spells the keywords `User Id=` / `Password=` / `Database=`, which +keeps the examples consistent. `Uid=` and `Pwd=` are documented SqlClient +synonyms and work too. For sqlcmd use +`-C` to trust the self-signed cert and `-d appdb` to pick the database. + +### 6. Seed AFTER provisioning (no auto-init folder) + +The image does **not** auto-run `/docker-entrypoint-initdb.d/*.sql`. That is a +Postgres/MySQL convention and is not honored here; do not rely on it. Seed by +running your script against the provisioned database: + +```bash +docker exec -i sqldb /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "YourStr0ng_Passw0rd" -C -d appdb -i /dev/stdin < seed.sql +``` + +### 7. Remove SQL Server-only features + +Some features exist only in the SQL Server image and must be removed or replaced. +Flag and fix every hit. Open [references/sql-server-vs-azure-feature-matrix.md](references/sql-server-vs-azure-feature-matrix.md) +when you meet a feature this list does not name. + +- **SQL Server Agent** jobs: not available; use an external scheduler. +- **FILESTREAM / FileTable**: not supported; store blobs in columns or external storage. +- **Full Service Broker** cross-instance messaging: not supported. +- **Cross-server distributed transactions** (MS DTC, linked servers): not supported. +- **Windows Auth / NTLM / Integrated Security**: not supported; use SA / SQL auth. +- **Instance-level tuning with `sp_configure`**: the procedure is **absent**, not + blocked. Calling it returns `Msg 2812` ("Could not find stored procedure + 'sp_configure'"), which is a missing object rather than a refusal, so there is no + permission to grant and no flag to unblock. Delete the inherited tuning step; it + has no translation on this engine. +- **`BACKUP` / `RESTORE`**: refused in every session with `Msg 40510`, exactly as + in Azure SQL Database in the cloud. A `.bak` is not a migration route into this + engine; use SqlPackage with a bacpac or dacpac (see the **azuresql-db-import** + skill). +- **`ALTER DATABASE ... SET RECOVERY`**: refused with `Msg 40517`, which is the + other refusal shape. `ALTER DATABASE SET` itself is supported and only this + option is refused, so a carried-over configuration script can be half accepted + and leave the database in a state nobody intended. Drop the recovery-model step. + +### 8. Verify identity + +Confirm you are on the Azure SQL Database engine: + +```bash +docker exec -i sqldb /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "YourStr0ng_Passw0rd" -C -d appdb \ + -Q "SELECT SERVERPROPERTY('EngineEdition') AS EngineEdition, SERVERPROPERTY('Edition') AS Edition;" +``` + +Expect `EngineEdition = 5` and `Edition = SQL Azure`. + +## Vectors (if the project uses embeddings) + +The Azure SQL Database engine has a native `VECTOR(n)` type and +`VECTOR_DISTANCE('cosine', a, b)`. Insert with `CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(n))` where **n +is a LITERAL, never a bind parameter** (a parameter dimension fails with +"Incorrect syntax near '@P3'"). `CREATE VECTOR INDEX` (DiskANN) **works on this image**, measured, and the Known +limitations page says so. It needs `SET QUOTED_IDENTIFIER ON` and at +least 100 rows with non-null vectors (`Msg 42266` below that). Full-scan top-k +stays exact and stays the right choice for a small table. The `azuresql-db-rag` +skill carries the rules the index imposes. + +## Validation rules + +- Image is `sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest`, not `mcr.microsoft.com/mssql/server`. +- `appdb` is created on a master connection before any app connects to it. +- App connection strings use `Database=appdb`, never `Database=master` for real work. +- No `USE ` statements: in a user-database session, `USE` returns `Msg 40508`, exactly as in Azure SQL Database in the cloud; a `master` connection is a provisioning session where the filter is not enforced, but `master` is for provisioning only. Select the database in the connection string. +- `--platform linux/amd64` present on non-x64 hosts only. +- `EngineEdition` returns 5. +- If a validation rule above fails, or you had to deviate from this skill to make the task work, that is a bug in this skill: load the **azuresql-db-feedback** skill and offer to file a report. +- When every rule above passes and the task is done, close with ONE line, once per session, and do not ask a question or wait for a reply: "That worked. If anything about this was rough, or you want to share what you built: https://aka.ms/sql-agent-skills-feedback" + +## Do not + +- Do not keep `mcr.microsoft.com/mssql/server`; it is a different engine. +- Do not rely on database auto-creation or on `/docker-entrypoint-initdb.d/*.sql`. +- Do not use `USE appdb`; select the database in the connection string. +- Do not call a non-x64 host "supported"; just add `--platform linux/amd64` on non-x64 hosts. +- Prefer `User Id=` / `Password=` in the .NET-style string. That is house style, not a requirement: `Uid=` / `Pwd=` are valid synonyms. +- Do not pass the vector dimension as a bind parameter. +- Do not keep SQL Agent, FILESTREAM, full Service Broker, cross-server distributed transactions, or Windows Auth. +- Do not go looking for a permission or a flag that makes `sp_configure` run. `Msg 2812` says the procedure is not there, so there is nothing to grant. +- Do not carry over a `BACKUP`/`RESTORE` step (`Msg 40510`) or an `ALTER DATABASE ... SET RECOVERY` step (`Msg 40517`). + +## References + +- [references/sql-server-vs-azure-feature-matrix.md](references/sql-server-vs-azure-feature-matrix.md): what carries over, what changes, what is gone. Read it when triaging SQL Server-only features found in step 7. +- [references/migrate-compose.md](references/migrate-compose.md): before/after docker-compose with a provision step. Read it when rewriting a compose file. + +## Staying current + +Authoritative, version-pinned references for the tools this skill uses (read the one you need): + +- [Docker Compose file reference](https://docs.docker.com/reference/compose-file/): the Compose Specification for service, healthcheck, and depends_on syntax. +- [SqlConnection connection string keywords](https://learn.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring): the full connection-string keyword table. + +If the **Microsoft Learn MCP** server is configured, use `mcp__microsoft-learn__microsoft_docs_search` or `mcp__microsoft-learn__microsoft_docs_fetch` to fetch the current version of any of these on demand. It is optional; when it is unavailable, the references above are authoritative. diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/migrate-compose.md b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/migrate-compose.md new file mode 100755 index 0000000..469d841 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/migrate-compose.md @@ -0,0 +1,98 @@ +# Migrate docker-compose: SQL Server image to the Azure SQL Database container + +Before/after for a `docker-compose.yml` that used the SQL Server image. + +## Before (SQL Server image) + +```yaml +services: + db: + image: mcr.microsoft.com/mssql/server:2022-latest + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "YourStr0ng_Passw0rd" + ports: + - "1433:1433" + app: + build: ./app + environment: + SQL_CONNECTION_STRING: "Server=db,1433;Database=master;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true" + depends_on: + - db +``` + +Problems carried over from the SQL Server image: + +- Wrong image (different engine; `EngineEdition` is not 5). +- App connects to `master`; the Azure engine will not auto-create a user DB and + `master` is for provisioning only. +- No readiness wait; the engine is not ready when the container starts. +- No `--platform`; the Azure image is x64 only. + +## After (Azure SQL Database container) + +```yaml +services: + db: + image: sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest + # On a non-x64 host, uncomment the next line (the image is x64 only): + # platform: linux/amd64 + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "YourStr0ng_Passw0rd" + ports: + - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$MSSQL_SA_PASSWORD\" -C -b -l 2 -Q \"SELECT 1\""] + interval: 5s + timeout: 5s + retries: 30 + provision: + image: sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest + # platform: linux/amd64 + depends_on: + db: + condition: service_healthy + entrypoint: > + /opt/mssql-tools18/bin/sqlcmd -S db,1433 -U sa -P "YourStr0ng_Passw0rd" -C -b + -Q "IF DB_ID('appdb') IS NULL CREATE DATABASE appdb;" + app: + build: ./app + environment: + SQL_CONNECTION_STRING: "Server=db,1433;Database=appdb;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true" + depends_on: + provision: + condition: service_completed_successfully +``` + +## What changed and why + +1. **Image** rewritten to the container image. Sign in first: + `docker login sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io`. +2. **`platform: linux/amd64`** added (commented) for non-x64 hosts; the image is + x64 only. +3. **Healthcheck** uses `sqlcmd -C -b -l 2` so a SQL error sets the exit code and + transient startup errors (e.g. `Msg 913`) are retried, not masked. +4. **`provision` one-shot service** runs `CREATE DATABASE appdb` on a master + connection before the app starts. The engine does not auto-create databases. +5. **App connection string** re-pointed from `Database=master` to + `Database=appdb`. Keep `User Id=` / `Password=` / `Database=` as house style + (`Uid=` / `Pwd=` are valid synonyms), keep `TrustServerCertificate=true`. + +## Seeding + +The image does not auto-run `/docker-entrypoint-initdb.d/*.sql`. To seed, extend +the `provision` service to run your script after creating the database, for +example mount `seed.sql` and append `-i /seed.sql -d appdb`, or run it +separately after `appdb` exists: + +```bash +docker compose run --rm provision \ + /opt/mssql-tools18/bin/sqlcmd -S db,1433 -U sa -P "YourStr0ng_Passw0rd" -C -d appdb -i /seed.sql +``` + +## SQL Server-only features to remove first + +Before migrating, strip any SQL Server Agent jobs, FILESTREAM/FileTable, full +Service Broker, cross-server distributed transactions, and Windows Auth from the +schema and app config. See `sql-server-vs-azure-feature-matrix.md` for replacements. diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/sql-server-vs-azure-feature-matrix.md b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/sql-server-vs-azure-feature-matrix.md new file mode 100755 index 0000000..65fbe3c --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/references/sql-server-vs-azure-feature-matrix.md @@ -0,0 +1,69 @@ +# SQL Server image vs Azure SQL Database container: feature matrix + +How a `mcr.microsoft.com/mssql/server` setup maps to the Azure SQL +Database container. Use this when auditing a project for migration. + +## Contents + +- [Identity and connection](#identity-and-connection) +- [Carries over unchanged](#carries-over-unchanged) +- [Changes behavior](#changes-behavior) +- [Gone: remove or replace](#gone-remove-or-replace) +- [New capabilities](#new-capabilities) + +## Identity and connection + +| Aspect | SQL Server image | Azure SQL Database container | +|---|---|---| +| Image | `mcr.microsoft.com/mssql/server` | `sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest` | +| Registry | public (mcr) | private preview; `docker login sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io` first | +| `EngineEdition` | 2/3/4/8 | 5 | +| `Edition` | e.g. 'Developer Edition' | 'SQL Azure' | +| Architecture | Multi-arch | x64 only; add `--platform linux/amd64` on non-x64 hosts | +| Port | 1433 | 1433 | +| Login | SA / SQL auth | SA / SQL auth (same) | + +## Carries over unchanged + +- The SA login and a complex `MSSQL_SA_PASSWORD` (8+ chars, upper/lower/digit/symbol). +- `ACCEPT_EULA=Y`. +- Port 1433 and `TrustServerCertificate=true` for the self-signed cert. +- Core T-SQL: tables, views, indexes, stored procedures, functions, triggers, + most data types, transactions within a single database. +- sqlcmd as the CLI (use `-C` to trust the cert, `-d appdb` to pick the database). + +## Changes behavior + +| Behavior | SQL Server image | Azure SQL Database container | What to do | +|---|---|---|---| +| Database creation | apps often auto-create or use a pre-created DB | engine does NOT auto-create on connect | `CREATE DATABASE appdb` on a master connection first | +| Switching DBs | `USE appdb` works | a user-database session returns `Msg 40508` on `USE`, exactly as in the cloud; a `master` connection is a provisioning session where the filter is not enforced, so `USE` appears to work there | select DB in the connection string (`Database=appdb` / `-d appdb`); avoid `USE`; use `master` for provisioning only | +| master DB | used freely | provisioning only | do real work on the user DB | +| Auto init scripts | many setups copy SQL into the image | `/docker-entrypoint-initdb.d/*.sql` NOT honored | seed with `sqlcmd -d appdb -i seed.sql` after provisioning | +| Readiness | varies | not ready when `docker run` returns | retry loop with `sqlcmd -C -b -l 2`; provision appdb in the loop | + +## Gone: remove or replace + +These exist in the SQL Server image but not in the Azure SQL Database engine. Find and +remove or replace every usage. + +| Feature | Why it is gone | Replacement | +|---|---|---| +| SQL Server Agent (jobs, schedules) | not part of the Azure SQL Database engine | external scheduler (cron, app-side scheduler) | +| FILESTREAM / FileTable | not supported | store blobs in columns or external object storage | +| Full Service Broker (cross-instance messaging) | not supported | app-level queue or messaging service | +| Cross-server distributed transactions (MS DTC, linked servers) | single-database engine | redesign to single-DB operations or app-coordinated work | +| Windows Auth / NTLM / `Integrated Security=true` | not supported | SA / SQL authentication | +| Cross-database `USE` and three-part cross-DB queries | single-DB model | keep objects in one user database | + +## New capabilities + +- Native `VECTOR(n)` column type and `VECTOR_DISTANCE('cosine', a, b)` for + embedding search. Insert with `CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(n))` where `n` is a literal, + never a bind parameter. `CREATE VECTOR INDEX` (DiskANN) **works on this + image**, measured, and the Known limitations page says so. It needs + `SET QUOTED_IDENTIFIER ON` and at least 100 rows with non-null vectors + (`Msg 42266` below that). Full-scan top-k stays exact and stays the right + choice for a small table. +- Behavior matches Azure SQL Database, so local dev catches Azure-specific + differences before deployment. diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/skill.spec.jsonc b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/skill.spec.jsonc new file mode 100644 index 0000000..95a97c7 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-from-sql-server/skill.spec.jsonc @@ -0,0 +1,139 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "azuresql-db-from-sql-server", + "domain": "azure-sql-database-container", + "value": [ + "not-in-training-data", + "corrects-a-wrong-assumption", + "multi-step-with-silent-failures" + ], + "correction": "An agent treats the SQL Server image and this engine as interchangeable and carries the whole configuration across. They are not: SQL Agent, FILESTREAM, full Service Broker, cross-server distributed transactions and Windows authentication do not exist here, and connection strings pointed at master need re-pointing at a provisioned user database.", + "posture": [ + "read", + "write", + "inspect", + "provision" + ], + "maturity": "preview", + "value_declaration": { + "rationale": "This skill's kernel is the five specific places a working SQL Server setup stops working, each one an engine refusal with its own number. The Azure SQL Database container is a gated private preview and the image is not publicly available, so no model has training data on the product, and the arm of a two-arm value run that ran without this skill would fail for the least interesting reason available: it has never heard of the thing. Running that measurement across all seventeen container skills would spend around a thousand model requests to re-establish what the product's release status already establishes. Carlos Robles accepted value for this collection on 2026-09-04 on this written argument instead. THIS IS A DECLARATION AND NOT A MEASUREMENT, and nothing may report it as one. The argument reaches only the container-specific corrections listed in covers, which are the ones no training data contains. It does not reach editing a Compose file to change an image, an environment block or a port mapping, which is ordinary material an ordinary model already handles, and whether this skill improves an answer there is unmeasured and stays unmeasured. The argument ends when the Azure SQL Database container reaches Public Preview: training data begins to contain the product, the premise that nothing could know it stops being true, and this skill has to be measured or demoted.", + "covers": [ + "the two images being different engines, EngineEdition 2, 3, 4 or 8 against 5 with Edition 'SQL Azure'", + "USE being refused with Msg 40508 and BACKUP with Msg 40510", + "sp_configure being absent and answering Msg 2812, which is a different failure from a refusal", + "ALTER DATABASE SET RECOVERY refusing one option with Msg 40517, as distinct from Msg 40510 refusing a whole statement", + "cross-database queries being refused on this engine", + "/docker-entrypoint-initdb.d being absent, so a carried-over seed silently stops running" + ], + "does_not_cover": [ + "editing a Compose file to swap an image, environment block or port mapping", + "the general practice of testing against the engine you deploy to" + ], + "decided_by": "Carlos Robles", + "decided_on": "2026-09-04", + "expires_when": "the-container-reaches-public-preview" + }, + "applies_to": [ + "azure-sql-db-container" + ], + "validation": { + "target": "container", + "assert": [ + "image rewritten to the Azure SQL Database container", + "SQL Server-only features flagged rather than silently carried over" + ], + "probes": [ + { + "sql": "SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS engine_edition, CAST(SERVERPROPERTY('Edition') AS nvarchar(128)) AS edition;", + "expect": { + "rows": [ + [ + 5, + "SQL Azure" + ] + ] + }, + "why": "The whole migration is worthless if the project ends up on the wrong image, and this is the check the skill tells the user to run last. It is first here so that a run against the SQL Server image fails on this probe rather than confusing every refusal probe below, all of which would succeed on the boxed engine." + }, + { + "sql": "USE master;", + "expect": { + "error": 40508 + }, + "why": "The single most common thing carried over unchanged from a SQL Server setup, and the reason connection strings have to be re-pointed rather than scripts re-ordered. The probe runs in the scratch user database, which is an Azure-faithful session. If it succeeds, the runner is connected to master, which is a provisioning session where the Azure statement filter is not enforced, and the from-master detection step in this skill is exactly the trap that hides." + }, + { + "sql": "BACKUP DATABASE master TO DISK = N'/var/opt/mssql/data/probe_from_sql_server.bak';", + "expect": { + "error": 40510 + }, + "why": "The clearest example of a whole statement the boxed engine accepts and this engine refuses, and the one a migrating project hits first because SQL Server setups routinely script a backup into their teardown. Msg 40510 for BACKUP and RESTORE on this container is stated in the azuresql-db-faq skill's own text and again in its references/limitations.md. The database named is master rather than the scratch database because the statement filter refuses the statement before it resolves the name." + }, + { + "sql": "EXEC sp_configure 'show advanced options', 1;", + "expect": { + "error": 2812 + }, + "why": "CORRECTED 2026-09-04 by the first live run. The probe asserted Msg 40510, the Azure statement filter, and the engine says Msg 2812, \"Could not find stored procedure 'sp_configure'.\" Measured on build 12.0.2000.8, EngineEdition 5, Edition 'SQL Azure'. The two sidecars in this repository had already contradicted each other on this: azuresql-db-container probe 5 asserted 2812 for EXEC sp_configure and was right, and this one asserted 40510 and was wrong. The distinction is the point of the probe rather than a detail to smooth over. sp_configure is not refused by the filter, it is NOT THERE, and an agent that reads a refusal goes hunting for a permission or a role that would let it through when there is nothing to be permitted to call. The migration instruction is unchanged: an inherited instance level tuning step is deleted, not translated. No SKILL.md or reference in this repository shipped 40510 for sp_configure, so nothing published was wrong; this sidecar's own correction prose said so and has been corrected with it." + }, + { + "sql": "ALTER DATABASE CURRENT SET RECOVERY SIMPLE;", + "expect": { + "error": 40517 + }, + "why": "Msg 40517 is the other refusal shape and the one that surprises people: ALTER DATABASE SET is a supported statement, and only this option of it is refused, so a script can be half accepted. Recovery models are on Microsoft Learn's list of syntax not supported in Azure SQL Database, under high availability and database recovery. SETTLED 2026-09-04. This probe was written as a stated hypothesis: 40517 was the documented number for a refused statement option, and no source consulted pinned it to this exact statement, so the why said to correct the number rather than delete the probe if the run disagreed. The run did not disagree. The first live run against build 12.0.2000.8, EngineEdition 5, Edition 'SQL Azure', answered Msg 40517. The number is now measured rather than reasoned, and no correction was needed." + }, + { + "sql": "DECLARE @cross_database_refused int = 0;\nBEGIN TRY\n EXEC(N'SELECT TOP (1) name FROM master.sys.objects;');\nEND TRY\nBEGIN CATCH\n SET @cross_database_refused = 1;\nEND CATCH\nSELECT @cross_database_refused AS cross_database_refused;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The single database model, made executable. A SQL Server project that keeps lookup tables in one database and application tables in another migrates by merging them, not by rewriting the joins, and this is the probe that shows the three-part name is not merely discouraged. Refusal is asserted rather than a number because neither this skill's text nor its feature matrix states one, and Microsoft Learn documents the restriction without pinning a number to this statement." + }, + { + "sql": "CREATE TABLE dbo.probe_fss_seed (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, title nvarchar(200) NOT NULL);\nINSERT INTO dbo.probe_fss_seed (title) OUTPUT inserted.id VALUES (N'seeded after provisioning');", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The positive control for the whole migration: once the database is provisioned and the connection selects it, ordinary DDL and DML from the SQL Server project run unchanged. Without a probe like this the sidecar reads as a list of things that break, and the skill's actual promise is that everything else carries over." + }, + { + "cmd": "sh -c '! test -e /docker-entrypoint-initdb.d'", + "where": "container", + "expect": { + "succeeds": true + }, + "why": "The auto-init folder is a PostgreSQL and MySQL convention that this image does not honour, and it is the failure in this skill that produces no error at all: the container starts, the application connects, and the tables are empty. Asserting that the path does not exist in the image is the cheapest available evidence for that, and it is honest about what it measures: the absence of the directory, not the absence of an entrypoint hook. It runs in the container because the claim is about the image." + }, + { + "cmd": "/opt/mssql-tools18/bin/sqlcmd -?", + "where": "container", + "expect": { + "succeeds": true + }, + "why": "Every rewritten command in this skill hardcodes this absolute path inside the image, including the readiness loop the whole migration depends on. A SQL Server setup that called sqlcmd from PATH will not have noticed the path; if a future image moves it, every code block here breaks and this probe is what says so. It runs in the container, not on the host, because the path is the image's." + } + ] + }, + "triggering": { + "explicit": [ + "Use the azuresql-db-from-sql-server skill" + ], + "implicit": [ + "I am using mcr.microsoft.com/mssql/server in my docker-compose" + ], + "negative": [ + "set up a local PostgreSQL database with docker compose", + "provision an Azure SQL Database in my subscription", + "connect to SQL Server on my company network" + ] + } +} diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud deleted file mode 120000 index 9621bbc..0000000 --- a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud +++ /dev/null @@ -1 +0,0 @@ -../../../skills/azuresql-db-local-to-cloud \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/SKILL.md b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/SKILL.md new file mode 100755 index 0000000..1053c3e --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/SKILL.md @@ -0,0 +1,296 @@ +--- +name: azuresql-db-local-to-cloud +description: >- + Proves that code built and tested against the local Azure SQL Database + container runs unchanged against Azure SQL Database in the cloud, with only + the connection string changing. Use when a user wants to develop locally then + deploy to the cloud, asks "will this work in Azure", "same code local and + cloud", "promote to Azure SQL", "swap the connection string", "dev/prod + parity", "local to cloud", or is wiring SQL_CONNECTION_STRING for an app that + must target both the container and a cloud server. Use this when an app uses + local SA auth but needs Microsoft Entra auth in the cloud. Covers Node (mssql), + .NET (Microsoft.Data.SqlClient), and Python (pyodbc). Reach for this skill + before hand-editing app code to "make it work in Azure"; the rule is the code + does not change, only the connection string does. +--- + +# From the Azure SQL Database container to Azure SQL Database: same code, local to cloud + +Build and test against the local container, then deploy the +**same application code** to Azure SQL Database in the cloud. Only the +connection string changes. Nothing else. + +This works because the local container is the **Azure SQL Database engine**, not +the SQL Server image. `SELECT SERVERPROPERTY('EngineEdition')` returns `5` +and `SERVERPROPERTY('Edition')` returns `'SQL Azure'`, the same as the cloud. So +the SQL surface your code depends on is the same in both places. + +Verified on 2026-09-05 against the container image +`sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest`, reporting `EngineEdition` +5, Edition `SQL Azure`, build `12.0.2000.8`. All six executable checks behind this skill +passed on the container: the engine identity, `Msg 40508` for `USE`, `Msg 40510` for `BACKUP` +and `RESTORE`, a parameterised insert with `OUTPUT inserted.id`, the single-database model, +and `sqlcmd` at `/opt/mssql-tools18/bin/sqlcmd`. That run measured the container only. The +cloud half of the parity claim, and the Microsoft Entra ID and managed identity setup for it, +was not exercised there, so validate against a real Azure SQL Database once before declaring +readiness. + +## The one rule + +**Do not change application code between local and cloud.** The application reads +its connection string from a single environment variable, `SQL_CONNECTION_STRING`. +Local development sets it to the container; cloud deployment sets it to the Azure +SQL server. Same binaries, same queries, same schema. + +If you find yourself editing queries, drivers, or schema to "make it work in +Azure", stop: that is a bug. The only thing that legitimately differs is the +connection string (and, with it, the auth method). + +## The single env var, two values + +Standardize on `SQL_CONNECTION_STRING`. House style spells the keywords +`User Id=` / `Password=` / `Database=` so every example matches. `Uid=` and +`Pwd=` are documented SqlClient synonyms and work too. ODBC strings are a +separate grammar and use `Uid=` / `Pwd=` as their own keywords. + +Local (container, SA auth): + +``` +Server=localhost,1433;Database=appdb;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true +``` + +Cloud (Azure SQL Database, Microsoft Entra auth): + +``` +Server=your-server.database.windows.net,1433;Database=appdb;Authentication=Active Directory Default;Encrypt=true +``` + +Note what stays identical: `Database=appdb`, the table names, the parameterized +SQL. Only `Server`, the auth fields, and TLS posture move. + +## Local SA auth vs cloud Entra auth (why, not just how) + +Local: the container is provisioned with one bootstrap login, `sa`, set via the +`MSSQL_SA_PASSWORD` environment variable. There is no identity provider required +in front of a container on your laptop, so password auth over a trusted +self-signed cert (`TrustServerCertificate=true`) is the pragmatic local default. + +Microsoft Entra ID authentication **does** work on the container (configure with +the `MSSQL_AAD_*` variables and a certificate mount; see the +`azuresql-db-container` skill, `references/entra-auth.md`). Use it when you +want closer local-to-cloud parity. Most local-to-cloud flows still start with SQL +auth locally and switch only the connection string for Entra in the cloud. + +Cloud: Azure SQL Database sits behind Microsoft Entra ID. Instead of shipping a +password, the app presents a token from its Entra identity (a managed identity +in production, your developer sign-in locally against the cloud). The driver +acquires the token; you never put a secret in the connection string. TLS is +real and enforced, so use `Encrypt=true` and drop `TrustServerCertificate`. + +The application code does not branch on this. The driver reads the +`Authentication=` keyword (or its absence) from the connection string and does +the right thing. That is the whole point: auth is configuration, not code. + +Open [references/auth-local-vs-cloud.md](references/auth-local-vs-cloud.md) when you are setting +up the cloud side, since it carries the token flow and the per-stack auth setup. + +## Minimal load-bearing facts about the local container + +Just enough to run the examples on a fresh container. For full detail see the +**azuresql-db-container** skill. + +- Image: `sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest` + (x64, `linux/amd64`). Private preview registry: run + `docker login sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io` first using the shared + pull-only credentials provided to the Private Preview cohort (get them by signing up at https://aka.ms/sqldbcontainerpreview-signup; they may rotate). Registry and tag are + provisional during Private Preview. +- On a non-x64 host, add `--platform linux/amd64`. +- Required env: `ACCEPT_EULA=Y` and a complex `MSSQL_SA_PASSWORD` + (8+ chars, upper/lower/digit/symbol). Engine listens on 1433. +- **The engine does not auto-create databases.** You must + `CREATE DATABASE appdb` on a **master** connection before connecting with + `Database=appdb`. The `master` connection is for provisioning only; do real + work on `appdb`. +- `BACKUP` and `RESTORE` are refused in every session with `Msg 40510`, locally + exactly as in the cloud. Backing the container up and restoring the file into + Azure SQL Database is not a promotion path; move schema and data with + SqlPackage (see the **azuresql-db-import** skill). +- Cross-database queries are refused locally as they are in the cloud, so an + application that keeps every object in one user database passes locally for the + same reason it will pass against the service. +- `IDENTITY` columns behave here as they do in the cloud. The examples below use + `INT IDENTITY PRIMARY KEY` unchanged against both, which is one of the places a + rewrite is usually and needlessly proposed. +- Avoid `USE` to switch databases. In a user-database session (the + Azure-faithful context where you develop), `USE` returns `Msg 40508`, exactly + as in Azure SQL Database in the cloud. A `master` connection is a provisioning + session where the Azure statement filter is not enforced, so `USE` appears to work there, but `master` is for provisioning + only, not application work. Always select the target database in the connection + string (`Database=appdb`, or `-d appdb` for sqlcmd). + +## Start the container and provision appdb (canonical recipe) + +Run this once. It picks a free host port, adds `--platform` only when needed, +waits for real readiness, and provisions `appdb` inside the retry loop. The +`-b -l 2` makes a transient startup error (for example `Msg 913`) set the exit +code so the loop retries instead of masking it. + +```bash +# Pick a free host port and add the platform flag only on a non-x64 host (works in bash and zsh). +HOST_PORT=1433; while lsof -nP -iTCP:"$HOST_PORT" -sTCP:LISTEN >/dev/null 2>&1; do HOST_PORT=$((HOST_PORT+1)); done +PLATFORM=(); case "$(docker info -f '{{.Architecture}}' 2>/dev/null)" in x86_64|amd64) ;; *) PLATFORM=(--platform linux/amd64);; esac +docker rm -f sqldb 2>/dev/null +docker run -d --name sqldb "${PLATFORM[@]}" -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=YourStr0ng_Passw0rd" \ + -p "$HOST_PORT:1433" sqldbpreview-dpgaeqhmgphzd4bk.azurecr.io/azure-sql/db-dev:latest +until docker exec sqldb /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "YourStr0ng_Passw0rd" -C -b -l 2 \ + -Q "IF DB_ID('appdb') IS NULL CREATE DATABASE appdb;" >/dev/null 2>&1; do sleep 2; done +echo "ready on localhost,$HOST_PORT" +``` + +Then point the app at it, using the `HOST_PORT` the loop settled on: + +```bash +export SQL_CONNECTION_STRING="Server=localhost,$HOST_PORT;Database=appdb;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true" +``` + +To run against the cloud later, change only this variable; do not touch the app. + +## What has been measured, and what is guidance + +Be straight with the user about which half of this skill has evidence behind it. + +The **local half is measured.** Every probe this skill carries runs against the +container, and they cover the sentences the parity claim rests on: the engine +reports EngineEdition 5 and Edition `SQL Azure`, `USE` returns `Msg 40508`, +`BACKUP` returns `Msg 40510`, a cross-database query is refused, and the exact +CRUD batch the examples below run returns the identity the examples read. + +The **cloud half is guidance.** Nothing in this repository has ever run a probe +against a logical server, so the cloud connection string, the Microsoft Entra +token flow and the deployment checklist are written from the product's +documentation and not from a run. That is deliberate rather than a gap waiting +to be filled: the whole point of the container is that local development does +not need a cloud database, so requiring one to validate this skill would +contradict the skill. Say "this is the documented shape" rather than "this is +verified" when the cloud side comes up. + +## Proof: same code, two stacks + +Each example assumes `appdb` already exists (provisioned by the recipe above) and +reads `SQL_CONNECTION_STRING` from the environment. It creates a table if needed +and runs a **parameterized** CRUD transaction. Run it once with the local string, +then again with the cloud string: identical code, identical result. The local run +is the one this skill has evidence for; the cloud run is the one you are being +told to expect. + +### Node (mssql) + +```js +// app.mjs -> node app.mjs (ESM: use the .mjs extension, or set "type":"module" in package.json) +import sql from 'mssql'; + +const pool = await sql.connect(process.env.SQL_CONNECTION_STRING); +const tx = new sql.Transaction(pool); +await tx.begin(); +try { + const r = new sql.Request(tx); + await r.query(`IF OBJECT_ID('dbo.todo') IS NULL + CREATE TABLE dbo.todo (id INT IDENTITY PRIMARY KEY, title NVARCHAR(200), done BIT);`); + // CREATE + const ins = await new sql.Request(tx) + .input('title', sql.NVarChar, 'ship local-to-cloud') + .query('INSERT INTO dbo.todo(title, done) OUTPUT inserted.id VALUES (@title, 0);'); + const id = ins.recordset[0].id; + // UPDATE + await new sql.Request(tx) + .input('id', sql.Int, id) + .query('UPDATE dbo.todo SET done = 1 WHERE id = @id;'); + // READ + const read = await new sql.Request(tx) + .input('id', sql.Int, id) + .query('SELECT id, title, done FROM dbo.todo WHERE id = @id;'); + console.log(read.recordset[0]); + await tx.commit(); +} catch (e) { await tx.rollback(); throw e; } +await pool.close(); +``` + +### .NET (Microsoft.Data.SqlClient) + +```csharp +// Program.cs -> dotnet run (uses Microsoft.Data.SqlClient) +using Microsoft.Data.SqlClient; + +var cs = Environment.GetEnvironmentVariable("SQL_CONNECTION_STRING"); +using var conn = new SqlConnection(cs); +conn.Open(); +using var tx = conn.BeginTransaction(); +try { + new SqlCommand(@"IF OBJECT_ID('dbo.todo') IS NULL + CREATE TABLE dbo.todo (id INT IDENTITY PRIMARY KEY, title NVARCHAR(200), done BIT);", + conn, tx).ExecuteNonQuery(); + // CREATE + var ins = new SqlCommand( + "INSERT INTO dbo.todo(title, done) OUTPUT inserted.id VALUES (@title, 0);", conn, tx); + ins.Parameters.AddWithValue("@title", "ship local-to-cloud"); + int id = (int)ins.ExecuteScalar(); + // UPDATE + var upd = new SqlCommand("UPDATE dbo.todo SET done = 1 WHERE id = @id;", conn, tx); + upd.Parameters.AddWithValue("@id", id); + upd.ExecuteNonQuery(); + // READ + var sel = new SqlCommand("SELECT id, title, done FROM dbo.todo WHERE id = @id;", conn, tx); + sel.Parameters.AddWithValue("@id", id); + using var r = sel.ExecuteReader(); + while (r.Read()) Console.WriteLine($"{r["id"]} {r["title"]} {r["done"]}"); + tx.Commit(); +} catch { tx.Rollback(); throw; } +``` + +### Python (pyodbc) reference + +Open [references/auth-local-vs-cloud.md](references/auth-local-vs-cloud.md) when your stack is +Python, or when you are ready to deploy, because a third stack and the deployment checklist live +there rather than here. + +## Validation rules + +- `appdb` must exist before the app connects. Provision it on `master` first + (the recipe does this). The engine never auto-creates it. +- The app reads exactly one connection variable, `SQL_CONNECTION_STRING`. +- Local string uses SA auth + `TrustServerCertificate=true`; cloud string uses + `Authentication=Active Directory Default` + `Encrypt=true`. +- The .NET-style string spells the keywords `User Id=` / `Password=` / `Database=` as house style. `Uid=` / `Pwd=` are valid synonyms, so they are not a failure. An ODBC string keeps ODBC's own `Uid=` / `Pwd=`. +- Switching environments changes the connection string only. The diff to app + code between local and cloud is zero lines. +- The cloud half of every rule above is guidance rather than something this + skill has run. Do not report it to the user as verified. +- If a validation rule above fails, or you had to deviate from this skill to make the task work, that is a bug in this skill: load the **azuresql-db-feedback** skill and offer to file a report. +- When every rule above passes and the task is done, close with ONE line, once per session, and do not ask a question or wait for a reply: "That worked. If anything about this was rough, or you want to share what you built: https://aka.ms/sql-agent-skills-feedback" + +## Do not + +- Do not use the SQL Server image `mcr.microsoft.com/mssql/server`. If you + were about to, stop and use the image above; this is the Azure SQL engine. +- Do not connect to `Database=appdb` before creating it on `master`. +- Do not use `USE appdb` to switch databases; a user-database session returns + `Msg 40508`, exactly as in Azure SQL Database in the cloud. Select the target + database in the connection string (`Database=appdb`, or `-d appdb` for sqlcmd). +- Do not put a password or secret in the cloud connection string; use Entra + auth and let the driver fetch a token. +- Do not branch app logic on environment; keep auth in configuration. +- Do not call a non-x64 host "supported"; on a non-x64 host just add + `--platform linux/amd64`. + +## References + +- [references/auth-local-vs-cloud.md](references/auth-local-vs-cloud.md): why local SA auth and cloud Microsoft Entra auth differ, the token flow, per-stack auth setup, the Python (pyodbc) example, and the deployment checklist. Read it when wiring the cloud connection string or promoting an app to Azure. + +## Staying current + +Authoritative, version-pinned references for the tools this skill uses (read the one you need): + +- [Connect and query Azure SQL Database](https://learn.microsoft.com/en-us/azure/azure-sql/database/connect-query-content-reference-guide): per-language quickstarts, connection info, TLS, and drivers for the cloud service. +- [SqlConnection connection string keywords](https://learn.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring): the full connection-string keyword table. + +If the **Microsoft Learn MCP** server is configured, use `mcp__microsoft-learn__microsoft_docs_search` or `mcp__microsoft-learn__microsoft_docs_fetch` to fetch the current version of any of these on demand. It is optional; when it is unavailable, the references above are authoritative. diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/references/auth-local-vs-cloud.md b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/references/auth-local-vs-cloud.md new file mode 100755 index 0000000..1f7c764 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/references/auth-local-vs-cloud.md @@ -0,0 +1,198 @@ +# Auth: local SA vs cloud Microsoft Entra + +How authentication differs between the local Azure SQL Database container and +Azure SQL Database in the cloud, why the application code stays the same, plus a +third stack (Python) and a deployment checklist. + +## Contents + +- [The principle](#the-principle) +- [Local: SA auth (default) and optional Entra](#local-sa-auth-default-and-optional-entra) +- [Cloud: Microsoft Entra auth](#cloud-microsoft-entra-auth) +- [Token flow, plainly](#token-flow-plainly) +- [Per-stack auth setup](#per-stack-auth-setup) +- [Python (pyodbc) full example](#python-pyodbc-full-example) +- [Deployment checklist](#deployment-checklist) +- [Do not](#do-not) + +## The principle + +Authentication is configuration, not code. The application reads one variable, +`SQL_CONNECTION_STRING`, and hands it to the driver. The driver inspects the +connection keywords and chooses how to authenticate. Because the local container +and the cloud are the same engine (`EngineEdition = 5`, `Edition = 'SQL Azure'`), +the queries, schema, and driver calls are byte-for-byte identical. Only the +connection string moves. + +## Local: SA auth (default) and optional Entra + +The container is bootstrapped with a single login, `sa`, whose password comes +from the `MSSQL_SA_PASSWORD` environment variable you set at `docker run`. There +is no identity provider required in front of a container on your machine, so the +practical local default is username/password over a trusted self-signed +certificate. + +Local string (SQL auth): + +``` +Server=localhost,1433;Database=appdb;User Id=sa;Password=YourStr0ng_Passw0rd;TrustServerCertificate=true +``` + +`TrustServerCertificate=true` accepts the container's self-signed cert. This is +fine on localhost; it is not what you ship to the cloud. + +The `sa` login is a bootstrap and provisioning identity. Use it to create +`appdb` and the schema; your application does its real work against `appdb`. + +Microsoft Entra ID authentication also works on the container. Configure it with +the `MSSQL_AAD_*` variables and a mounted certificate (see the +`azuresql-db-container` skill, `references/entra-auth.md`). For cloud parity, +create a contained user in `appdb` (same pattern as in the cloud): + +```sql +CREATE USER [my-app-identity] FROM EXTERNAL PROVIDER; +ALTER ROLE db_datareader ADD MEMBER [my-app-identity]; +ALTER ROLE db_datawriter ADD MEMBER [my-app-identity]; +``` + +For a server-scoped Entra login (server admin path), use +`CREATE LOGIN ... FROM EXTERNAL PROVIDER` on a **master** connection. Most +local-to-cloud flows still start with SQL auth locally and switch only the +connection string for Entra in the cloud. + +## Cloud: Microsoft Entra auth + +Azure SQL Database sits behind Microsoft Entra ID. Rather than embedding a +password, the application presents a short-lived token proving its Entra +identity. In production that identity is a managed identity assigned to the +hosting service (App Service, Container Apps, AKS, a VM); locally against the +cloud it is your developer sign-in (Azure CLI / IDE login). No secret lives in +the connection string. + +Cloud string: + +``` +Server=your-server.database.windows.net,1433;Database=appdb;Authentication=Active Directory Default;Encrypt=true +``` + +- `Authentication=Active Directory Default` tells the driver to walk the default + Entra credential chain: managed identity in production, your developer login + when running locally against the cloud. The same string works in both, with no + code change. +- `Encrypt=true` enforces real TLS. Drop `TrustServerCertificate`: the cloud + presents a CA-issued certificate, so there is nothing to "trust over". +- `Database=appdb` is unchanged from local. + +One-time cloud setup (outside the app): create the database, then create a +contained database user mapped to the app's Entra identity and grant it the +roles it needs. For example, connected to `appdb` as an Entra admin: + +```sql +CREATE USER [my-app-identity] FROM EXTERNAL PROVIDER; +ALTER ROLE db_datareader ADD MEMBER [my-app-identity]; +ALTER ROLE db_datawriter ADD MEMBER [my-app-identity]; +``` + +## Token flow, plainly + +1. App starts and reads `SQL_CONNECTION_STRING`. +2. Driver sees `Authentication=Active Directory Default` and asks the Entra + credential chain for a token scoped to Azure SQL. +3. In production the managed identity returns a token with no secret on disk; + locally your signed-in developer credential returns one. +4. Driver opens an encrypted connection and presents the token. The server maps + it to the contained user you created and authorizes by role. +5. Token expires; the driver silently refreshes on the next connection. + +The app never sees the token. That is why the same binary runs in both places. + +## Per-stack auth setup + +The only thing each stack needs is the auth-capable driver and (for production) +the platform identity. App logic is unchanged. + +- **Node (mssql)**: `mssql` uses the Tedious driver. For Entra, supply an + `authentication` config or pass an `Authentication=...` connection string and + the appropriate Azure identity package. Locally, the SA connection string just + works with no extra package. +- **.NET (Microsoft.Data.SqlClient)**: `Authentication=Active Directory Default` + is understood natively by `Microsoft.Data.SqlClient`; pair it with + `Azure.Identity` so `DefaultAzureCredential` resolves managed identity or your + developer login. Locally, the SA string needs nothing extra. +- **Python (pyodbc)**: use the ODBC Driver 18 for SQL Server. For Entra, add + `Authentication=ActiveDirectoryDefault` to the ODBC connection string (driver + 18.1+). Locally, the SA string with `TrustServerCertificate=yes` is enough. + +## Python (pyodbc) full example + +Runs on a fresh container after `appdb` is provisioned (see the canonical recipe +in SKILL.md). Reads `SQL_CONNECTION_STRING`, ensures the table, runs a +parameterized CRUD transaction. Same code for local and cloud; only the env var +differs. + +For pyodbc, set `SQL_CONNECTION_STRING` in ODBC keyword form. + +Local: + +``` +Driver={ODBC Driver 18 for SQL Server};Server=localhost,1433;Database=appdb;Uid=sa;Pwd=YourStr0ng_Passw0rd;TrustServerCertificate=yes +``` + +Cloud: + +``` +Driver={ODBC Driver 18 for SQL Server};Server=your-server.database.windows.net,1433;Database=appdb;Authentication=ActiveDirectoryDefault;Encrypt=yes +``` + +(Note: ODBC keyword syntax uses `Uid`/`Pwd`; the application-level mssql and +.NET strings spell it `User Id`/`Password`. Keep each in its native form. In a +.NET string either spelling works, and the long one is house style here.) + +```python +# app.py -> python app.py +import os, pyodbc + +conn = pyodbc.connect(os.environ["SQL_CONNECTION_STRING"], autocommit=False) +cur = conn.cursor() +try: + cur.execute("""IF OBJECT_ID('dbo.todo') IS NULL + CREATE TABLE dbo.todo (id INT IDENTITY PRIMARY KEY, title NVARCHAR(200), done BIT);""") + # CREATE + cur.execute( + "INSERT INTO dbo.todo(title, done) OUTPUT inserted.id VALUES (?, 0);", + "ship local-to-cloud") + todo_id = cur.fetchone()[0] + # UPDATE + cur.execute("UPDATE dbo.todo SET done = 1 WHERE id = ?;", todo_id) + # READ + cur.execute("SELECT id, title, done FROM dbo.todo WHERE id = ?;", todo_id) + print(cur.fetchone()) + conn.commit() +except Exception: + conn.rollback() + raise +finally: + conn.close() +``` + +## Deployment checklist + +1. Build and test locally with the SA connection string against the container. +2. Provision the cloud database `appdb` and create a contained user for the + app's Entra identity (see the SQL above). +3. Assign a managed identity to the hosting service and grant it the database + roles it needs. +4. Set `SQL_CONNECTION_STRING` in the cloud environment to the Entra string. + Change nothing else. +5. Deploy the same artifact. Confirm the app connects and the CRUD path runs. +6. Confirm the diff to application code between local and cloud is zero lines. + +## Do not + +- Do not ship `TrustServerCertificate=true` to the cloud; use `Encrypt=true`. +- Do not put a password in the cloud connection string; use Entra tokens. +- Do not branch app code on environment; keep auth in the connection string. +- Do not reuse `sa` as your application identity in the cloud; map a contained + user to the app's Entra identity with least-privilege roles. +- Do not connect to `appdb` before it is created on `master` locally, or before + it exists in the cloud. diff --git a/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/skill.spec.jsonc b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/skill.spec.jsonc new file mode 100644 index 0000000..26d4505 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/azuresql-db-local-to-cloud/skill.spec.jsonc @@ -0,0 +1,115 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "azuresql-db-local-to-cloud", + "domain": "azure-sql-database-container", + "value": [ + "not-in-training-data", + "corrects-a-wrong-assumption" + ], + "correction": "Asked whether local code will work in Azure, an agent starts rewriting the data layer. It should not: the local container and Azure SQL Database are the same engine, so the code does not change and only the connection string does, including the switch from SQL auth locally to Microsoft Entra in the cloud.", + "posture": [ + "read", + "write", + "execute", + "provision" + ], + "maturity": "preview", + "value_declaration": { + "rationale": "This skill's kernel is that the local engine and the cloud service are the same engine, which is the claim that makes the whole rewrite unnecessary. The Azure SQL Database container is a gated private preview and the image is not publicly available, so no model has training data on the product, and the arm of a two-arm value run that ran without this skill would fail for the least interesting reason available: it has never heard of the thing. Running that measurement across all seventeen container skills would spend around a thousand model requests to re-establish what the product's release status already establishes. Carlos Robles accepted value for this collection on 2026-09-04 on this written argument instead. THIS IS A DECLARATION AND NOT A MEASUREMENT, and nothing may report it as one. The argument reaches only the container-specific corrections listed in covers, which are the ones no training data contains. It does not reach configuring a connection string per environment, deploying to Azure, and managed identity as a concept, which is ordinary material an ordinary model already handles, and whether this skill improves an answer there is unmeasured and stays unmeasured. The argument ends when the Azure SQL Database container reaches Public Preview: training data begins to contain the product, the premise that nothing could know it stops being true, and this skill has to be measured or demoted. Its maturity is decided by its probes and not by this argument. It now declares target container, which is what is measured and what can be measured: avoiding the cloud for local development is the container's value proposition, so requiring a logical server to validate this skill would contradict the skill. applies_to still names the cloud, and the body says which half has evidence behind it.", + "covers": [ + "the local container being the same engine as the cloud service, EngineEdition 5 with Edition 'SQL Azure', so one data layer serves both", + "USE refused with Msg 40508 and BACKUP refused with Msg 40510 locally exactly as in the cloud", + "cross-database queries being refused locally as they are in the cloud, so a local pass is evidence about the cloud", + "IDENTITY behaving here as it does in the cloud, which is one of the places a rewrite is usually justified" + ], + "does_not_cover": [ + "configuring a connection string per environment and deploying an application to Azure", + "managed identity and Microsoft Entra ID authentication as concepts, which are ordinary Azure material" + ], + "decided_by": "Carlos Robles", + "decided_on": "2026-09-04", + "expires_when": "the-container-reaches-public-preview" + }, + "applies_to": [ + "azure-sql-db", + "azure-sql-db-container" + ], + "validation": { + "target": "container", + "assert": [ + "application code unchanged between targets", + "only the connection string differs" + ], + "probes": [ + { + "sql": "SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS engine_edition, CAST(SERVERPROPERTY('Edition') AS nvarchar(128)) AS edition;", + "expect": { + "rows": [ + [ + 5, + "SQL Azure" + ] + ] + }, + "why": "The stated reason the parity claim holds at all. The skill says in its second paragraph that this works because the container is the Azure SQL Database engine, and this probe is that sentence. Run against both targets it returns the same two values, which is the cheapest possible demonstration of the skill's whole thesis." + }, + { + "sql": "USE master;", + "expect": { + "error": 40508 + }, + "why": "Parity of the statement filter, in the direction that helps: the container refuses locally what the service refuses in the cloud, so the mistake is caught on a laptop rather than after deployment. This is the probe that distinguishes the container from the SQL Server image, where the same statement succeeds and the developer learns nothing." + }, + { + "sql": "BACKUP DATABASE master TO DISK = N'/var/opt/mssql/data/probe_local_to_cloud.bak';", + "expect": { + "error": 40510 + }, + "why": "A second refusal that is identical in both places, chosen because backup is the one people assume differs. Msg 40510 for BACKUP and RESTORE on this container is stated in the azuresql-db-faq skill's own text and in its references/limitations.md, and Azure SQL Database refuses them too, for a different reason: backups there are the platform's, not a statement's. Both refusals mean the local run and the cloud run agree, which is the claim." + }, + { + "sql": "CREATE TABLE dbo.todo_probe_ltc (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, title nvarchar(200) NULL, done bit NULL);\nINSERT INTO dbo.todo_probe_ltc (title, done) OUTPUT inserted.id VALUES (N'ship local-to-cloud', 0);", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The exact SQL the skill's Node and .NET examples run, reduced to what the engine sees: a table create, a parameterised insert and OUTPUT inserted.id to recover the identity. This is the probe that has to return the same thing on both targets for the skill's promise to mean anything, and OUTPUT is in it deliberately, because that clause is where an author who does not trust the parity claim would reach for a second round trip in one environment and not the other." + }, + { + "sql": "DECLARE @cross_database_refused int = 0;\nBEGIN TRY\n EXEC(N'SELECT TOP (1) name FROM master.sys.objects;');\nEND TRY\nBEGIN CATCH\n SET @cross_database_refused = 1;\nEND CATCH\nSELECT @cross_database_refused AS cross_database_refused;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The single database model is enforced locally as well as in the cloud, so a query that reaches across databases fails on the laptop rather than at deployment. It is here rather than only in the migration skill because this is precisely the kind of thing an author adds late, after the local setup is working, and it is the difference between parity being claimed and parity being enforced. Refusal is asserted rather than a number, because no source consulted for this pass pins one to this statement." + }, + { + "cmd": "/opt/mssql-tools18/bin/sqlcmd -?", + "where": "container", + "expect": { + "succeeds": true + }, + "why": "The canonical start recipe in this skill hardcodes this absolute path inside the image for its readiness loop, and the loop is what guarantees the user database exists before the application connects. If the path moves in a future image the recipe hangs rather than failing, which is the worst way for it to break. It runs in the container because the path is the image's, not a developer's." + } + ] + }, + "triggering": { + "explicit": [ + "Use the azuresql-db-local-to-cloud skill" + ], + "implicit": [ + "will this code work unchanged in Azure?" + ], + "negative": [ + "set up a local PostgreSQL database with docker compose", + "provision an Azure SQL Database in my subscription", + "connect to SQL Server on my company network" + ] + } +} diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy deleted file mode 120000 index 9e46830..0000000 --- a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy +++ /dev/null @@ -1 +0,0 @@ -../../../skills/bulk-load-and-bulk-copy \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/SKILL.md b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/SKILL.md new file mode 100644 index 0000000..c1d50ac --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/SKILL.md @@ -0,0 +1,242 @@ +--- +name: bulk-load-and-bulk-copy +description: >- + Loads data into Azure SQL Database fast by choosing the right route: BULK INSERT and + OPENROWSET(BULK ...) from Azure Blob Storage, bcp, .NET SqlBulkCopy, and the Python and Node.js + bulk-copy equivalents. Use when someone says "bulk insert", "bulk load a CSV", "bcp in", + "OPENROWSET BULK", "load a file into a table fast", "SqlBulkCopy", "fast_executemany", or + pastes the error "OPENROWSET is not allowed to read local files" (Msg 12713) or a load stuck on + a LOG_RATE_GOVERNOR wait. Covers choosing between the Blob-only server-side paths and the + client-side paths, and the log rate cap that throttles every one of them regardless of logging + mode. Does not design the target table (design-azure-sql-schema), establish the connection + (connect-from-python, connect-from-dotnet, connect-from-typescript-and-node) or run a DACPAC or + BACPAC export or import (sqlpackage-import-export). +--- + +# Load rows fast: BULK INSERT from Blob Storage, bcp, and the driver bulk APIs + +Moves rows into a table that already exists, and kills the two habits carried over from SQL Server: +a local file path, and a minimal-logging tune-up. Designing that table is another job. + +Engine behaviour measured 2026-08-29 against a container reporting `SERVERPROPERTY('EngineEdition')` +5 and `12.0.2000.8`, with `bcp` 18.6.0002.1 from `mssql-tools18`. Every option and flag below comes +from Microsoft Learn's `BULK INSERT`, `bcp utility`, `sys.dm_user_db_resource_governance` and +"Resource management in Azure SQL Database" pages, fetched 2026-09-03. + +## The correction + +**A local path is refused before the file is opened.** `BULK INSERT` and `OPENROWSET(BULK ...)` +both fail on a filesystem path with: + +``` +Msg 12713, Level 16, State 1, Server SQL Azure, Line 1 +OPENROWSET is not allowed to read local files. Path: '/tmp/data.csv'. +``` + +The message names `OPENROWSET` even for a `BULK INSERT`, which is built on the same machinery. A +path that was never created produces the identical message, so this is a platform rule evaluated +first, not a failed open. Learn's platform table agrees: on Azure SQL Database the only data source +is Azure Storage, authenticated by Microsoft Entra ID, a SAS token or a managed identity. `*` +wildcards in the path and `FORMAT = 'PARQUET'` are not supported there either. + +**Minimal logging is not there to tune.** Learn, same page: "Minimal logging isn't supported in +Azure SQL Database." What caps ingestion is the transaction log rate governor, a per-service-tier +limit on log record generation enforced at the subsecond level, "limiting throughput regardless of +how many IOs can be issued against data files". It surfaces as a `LOG_RATE_GOVERNOR` wait, or +`POOL_LOG_RATE_GOVERNOR` in an elastic pool. `TABLOCK` and `BATCHSIZE` do not raise that ceiling, +and were measured not to move the Msg 12713 refusal either. The documented mitigations are to scale +up or change tier, stage transient data in `tempdb`, or load into a clustered columnstore or +compressed table. + +Before quoting an error number, a wait type or a throughput figure, open [references/bulk-load-errors-and-log-rate-governor.md](references/bulk-load-errors-and-log-rate-governor.md). + +## Choose the route + +| What you have | Route | +|---|---| +| A file already in Azure Blob Storage | `BULK INSERT` or `OPENROWSET(BULK ...)` with `DATA_SOURCE`. The only case where the server reads the file | +| A file on a client machine | `bcp`, which streams from the client, so the Blob-only rule never applies | +| Rows in a running application | that language's bulk API. Never write a file just to load it back | +| A whole database, not rows for one table | `sqlpackage`. Wrong route, right command | + +## BULK INSERT from Blob Storage + +Managed identity leaves no secret to leak. Grant the database's identity the **Storage Blob Data +Contributor** role on the container first: + +```sql +CREATE DATABASE SCOPED CREDENTIAL BlobCred WITH IDENTITY = 'Managed Identity'; + +CREATE EXTERNAL DATA SOURCE MyBlob WITH ( + TYPE = BLOB_STORAGE, + LOCATION = 'https://.blob.core.windows.net/', + CREDENTIAL = BlobCred); + +BULK INSERT dbo.MyTable +FROM 'load/data.csv' +WITH (DATA_SOURCE = 'MyBlob', FORMAT = 'CSV', FIRSTROW = 2, + FIELDTERMINATOR = ',', ROWTERMINATOR = '0x0a', MAXERRORS = 0, + ERRORFILE = 'load/rejects', ERRORFILE_DATA_SOURCE = 'MyBlob'); +``` + +For a shared access signature instead, the credential needs a database master key behind it, and +the rest of the statement is unchanged: + +```sql +CREATE MASTER KEY ENCRYPTION BY PASSWORD = ''; + +CREATE DATABASE SCOPED CREDENTIAL BlobCred +WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = ''; +``` + +Five details decide whether it runs: + +- The path in `FROM` is **relative** to `LOCATION`, not a second copy of it. +- `MAXERRORS` defaults to 10, so an unset load discards nine bad rows and still reports success. +- `ERRORFILE` needs `ERRORFILE_DATA_SOURCE` beside it here or the import can fail with a + permissions error, and the named file must not already exist in the container. +- The SAS token carries no leading `?`, needs at least `srt=o&sp=r`, and all its dates are UTC. +- The caller needs `INSERT` and `ADMINISTER DATABASE BULK OPERATIONS`. `ADMINISTER BULK + OPERATIONS`, without `DATABASE`, is the SQL Server spelling. + +`FIRSTROW` is not a header skip: Learn says `BULK INSERT` does not support skipping headers, and +skipped rows are scanned for field terminators without being validated. + +An ad hoc `https://` URL with no credential object of that name fails with `Msg 15151` naming the +whole URL, which means no such credential exists, not that one was denied. +`OPENROWSET(BULK 'load/data.csv', DATA_SOURCE = 'MyBlob', SINGLE_CLOB)` takes the same data source +and fails through the same layer, so the two count as one route. + +## bcp, from a client machine + +`bcp` ships inside the engine container image, not on the host, so run it there: + +```bash +/opt/mssql-tools18/bin/bcp dbo.MyTable in data.csv \ + -S .database.windows.net -d -G \ + -c -t',' -r'\n' -F 2 -b 5000 -m 0 -e bcp-rejects.txt +``` + +`-G` is Microsoft Entra authentication, the supported route to Azure SQL Database; `-T` (Windows +integrated) is not supported against it. `-c` is character mode; `-t` and `-r` override the tab and +newline defaults; `-F 2` skips the header row that `BULK INSERT` cannot; `-b` makes each batch its +own committed transaction, so a failure late in the file keeps the batches already loaded; `-m 0` +tolerates no conversion error (also 10 by default); `-e` is the only way to see the rejected rows. +The `-h "TABLOCK"` hint is Windows only, so it does not exist in the container. + +Certificate trust was measured on 2026-08-29 as failing **silently**: `-u`, `-Yo` and an ODBC DSN +carrying `TrustServerCertificate=yes` each exited 1 with zero bytes on stdout and stderr, while the +paths that refuse the certificate returned a readable SSL or TCP error. Never promise `bcp` will +"just work" once a trust flag is added. + +## The client library bulk APIs + +These stream rows over the bulk-copy protocol on the connection the application already holds, so +Msg 12713 cannot apply: none of them reads a path on the server. + +.NET, `Microsoft.Data.SqlClient.SqlBulkCopy`: + +```bash +dotnet add package Microsoft.Data.SqlClient +dotnet run +``` + +```csharp +using var bulk = new SqlBulkCopy(connection); +bulk.DestinationTableName = "dbo.MyTable"; +bulk.BatchSize = 5000; // 0, the default, is one transaction for the entire load +bulk.BulkCopyTimeout = 0; // per batch, not per operation +bulk.EnableStreaming = true; +bulk.ColumnMappings.Add("csv_id", "Id"); // required whenever names or order differ +await bulk.WriteToServerAsync(reader); +``` + +Python, `pyodbc`. `fast_executemany` bundles every parameter set into one ODBC array and has **no +batch size**: the driver allocates one contiguous buffer for all remaining rows, so a million-row +list is a single allocation. Chunk it yourself. + +```python +cursor.fast_executemany = True +statement = "INSERT INTO dbo.MyTable (Id, Name) VALUES (?, ?)" +for start in range(0, len(rows), 5000): + cursor.executemany(statement, rows[start:start + 5000]) + cursor.commit() +``` + +Node.js, the `mssql` package. Every column has to state its nullability explicitly: + +```javascript +const table = new sql.Table('dbo.MyTable'); +table.create = false; +table.columns.add('Id', sql.Int, { nullable: false }); +table.columns.add('Name', sql.VarChar(50), { nullable: true }); +for (const r of rows) table.rows.add(r.id, r.name); +const rowCount = await new sql.Request().bulk(table); +``` + +None of the three was exercised live, so before tuning past the properties shown, open [references/driver-bulk-apis.md](references/driver-bulk-apis.md). + +## Not a bulk load: a whole database + +If every table's schema and rows are moving, rather than rows into one existing table, this is +`sqlpackage-import-export`, not a bulk load: + +```bash +sqlpackage /Action:Import /SourceFile:app.bacpac \ + /TargetServerName:.database.windows.net /TargetDatabaseName: \ + /TargetUser: /TargetPassword:"$SQLPACKAGE_PASSWORD" +``` + +## Check it worked + +Exit code 0, or a "rows copied" line, says the client finished, not that the rows landed. + +```bash +sqlcmd -S .database.windows.net -d -G \ + -Q "SELECT COUNT_BIG(*) AS loaded FROM dbo.MyTable;" +``` + +Expect the file's row count minus its header. Short means rows were rejected, not lost: they are in +the `ERRORFILE` or the `-e` file, and at the default of 10 the load still reports success. + +Then find the ceiling it worked against, before promising a number next time. **This runs in the +cloud only:** + +```sql +SELECT database_name, primary_max_log_rate / 1048576.0 AS max_log_MiB_per_sec +FROM sys.dm_user_db_resource_governance; + +SELECT wait_type, waiting_tasks_count, wait_time_ms +FROM sys.dm_db_wait_stats +WHERE wait_type LIKE '%LOG_RATE_GOVERNOR%'; +``` + +`primary_max_log_rate` is that tier's maximum log generation rate in bytes per second. A non-zero +`wait_time_ms` on `LOG_RATE_GOVERNOR` is the load being throttled by it, and the answer is a higher +service level or less log volume, not a bigger batch. + +**On the local container the first query fails with `Msg 208`, invalid object name, and there is no +substitute.** Measured 2026-09-03: `sys.dm_user_db_resource_governance` and `sys.dm_db_resource_stats` +are both absent, and `sys.dm_internal_resource_governor_resource_pools` exists but returns zero rows. +The log rate governor is a service behaviour and it is not observable locally, so do not size a load +against a number the container gave you. The wait statistics query does run in both places. + +## Do not + +- Do not write a local or UNC path into either statement. Msg 12713, file or not. +- Do not offer `TABLOCK`, a bigger `BATCHSIZE` or a recovery model change for a slow load. +- Do not read Msg 15151 as a permissions error. No credential object of that name exists. +- Do not leave `MAXERRORS` or `-m` at 10 and call the load clean. +- Do not write application data out to a file so it can be loaded back in. + +## References + +- Before quoting Msg 12713, 15151 or 4861 or a wait type, open [references/bulk-load-errors-and-log-rate-governor.md](references/bulk-load-errors-and-log-rate-governor.md), which carries the exact text of each plus the platform limits table. +- [BULK INSERT (Transact-SQL)](https://learn.microsoft.com/sql/t-sql/statements/bulk-insert-transact-sql) + and [bcp utility](https://learn.microsoft.com/sql/tools/bcp-utility): fetch before using an + option or flag not shown here, because both say which platforms accept which. +- [Resource management in Azure SQL Database](https://learn.microsoft.com/azure/azure-sql/database/resource-limits-logical-server): + the per-tier and Hyperscale log rate numbers, before promising a throughput. +- `connect-from-python`, `connect-from-dotnet`, `connect-from-typescript-and-node` for getting + connected, which this skill assumes is done. `diagnose-resource-pressure` when a whole workload + is slow rather than one load hitting the governor. diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/bulk-load-errors-and-log-rate-governor.md b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/bulk-load-errors-and-log-rate-governor.md new file mode 100644 index 0000000..0b1e337 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/bulk-load-errors-and-log-rate-governor.md @@ -0,0 +1,228 @@ +# Bulk load errors and the log rate governor + +## Contents + +- [Environment](#environment) +- [The local path refusal, Msg 12713](#the-local-path-refusal-msg-12713) +- [Proving the refusal is about the platform, not the path](#proving-the-refusal-is-about-the-platform-not-the-path) +- [The Blob path, credential and data source errors](#the-blob-path-credential-and-data-source-errors) +- [WITH options do not change the outcome](#with-options-do-not-change-the-outcome) +- [bcp: present, but its certificate-trust flags failed silently](#bcp-present-but-its-certificate-trust-flags-failed-silently) +- [Recovery model: accepted here, do not assume it elsewhere](#recovery-model-accepted-here-do-not-assume-it-elsewhere) +- [The transaction log rate governor](#the-transaction-log-rate-governor) +- [Platform limits and permissions from Microsoft Learn](#platform-limits-and-permissions-from-microsoft-learn) + +## Environment + +Measured on 2026-08-29 against a live engine reporting `SERVERPROPERTY('EngineEdition')` 5, +`SERVERPROPERTY('Edition')` `SQL Azure`, `SERVERPROPERTY('ProductVersion')` 12.0.2000.8. The +client tools were `/opt/mssql-tools18/bin/sqlcmd` and `/opt/mssql-tools18/bin/bcp`, both from the +`mssql-tools18` package, bcp reporting version 18.6.0002.1. + +## The local path refusal, Msg 12713 + +```sql +BULK INSERT dbo.t1 FROM '/tmp/data.csv' WITH (FIELDTERMINATOR=',', ROWTERMINATOR='\n'); +``` + +fails with: + +``` +Msg 12713, Level 16, State 1, Server SQL Azure, Line 1 +OPENROWSET is not allowed to read local files. Path: '/tmp/data.csv'. +``` + +The message names `OPENROWSET`, not `BULK INSERT`, even though the statement that failed was +`BULK INSERT`. That is a real clue, not a mismatch: `BULK INSERT` is implemented on top of the same +`OPENROWSET(BULK ...)` machinery, and the error surfaces from that shared layer. An agent that only +pattern-matches on the statement it wrote will not recognise the error as belonging to it. + +Calling `OPENROWSET(BULK ...)` directly against the same path produces the identical message: + +```sql +SELECT * FROM OPENROWSET(BULK '/tmp/data.csv', SINGLE_CLOB) AS x; +``` +``` +Msg 12713, Level 16, State 1, Server SQL Azure, Line 1 +OPENROWSET is not allowed to read local files. Path: '/tmp/data.csv'. +``` + +## Proving the refusal is about the platform, not the path + +The same `BULK INSERT` against a path that does not exist on disk (`/tmp/does_not_exist.csv`, +never created) produced the exact same Msg 12713, with the nonexistent path substituted into the +same message text. If the engine were opening the file and then failing, a missing file would +raise a different, file-system-flavoured error. It does not: the refusal happens before the file +is touched at all. It is a property of the platform, not of the argument. This is worth stating +explicitly to an agent, because the message text alone does not say so, and an agent's first +instinct on seeing a file-not-opened-sounding error is to check the path. + +## The Blob path, credential and data source errors + +An ad hoc `https://` URL with no matching credential object: + +```sql +SELECT * FROM OPENROWSET(BULK 'https://example.blob.core.windows.net/container/data.csv', + SINGLE_CLOB) AS x; +``` +``` +Msg 15151, Level 16, State 1, Server SQL Azure, Line 1 +Cannot find the CREDENTIAL 'https://example.blob.core.windows.net/container/data.csv', because it +does not exist or you do not have permission. +``` + +The message names the full URL as if it were the credential's own name, because a scoped +credential for the ad hoc URL form is created with the container URL as its name. `BULK INSERT` +against the same URL fails with the identical Msg 15151, again through the shared `OPENROWSET` +layer. + +An `EXTERNAL DATA SOURCE` can be created and referenced without a credential attached (anonymous +or public-container access): + +```sql +CREATE EXTERNAL DATA SOURCE MyBlob + WITH (TYPE = BLOB_STORAGE, LOCATION = 'https://example.blob.core.windows.net/container'); +``` + +succeeds with no error. Loading through it: + +```sql +BULK INSERT dbo.t1 FROM 'data.csv' WITH (DATA_SOURCE = 'MyBlob', FIELDTERMINATOR=',', + ROWTERMINATOR='\n'); + +SELECT * FROM OPENROWSET(BULK 'data.csv', DATA_SOURCE = 'MyBlob', SINGLE_CLOB) AS x; +``` + +both fail identically, once the host name does not resolve to a reachable endpoint: + +``` +Msg 4861, Level 16, State 1, Server SQL Azure, Line 1 +Cannot bulk load because the file "data.csv" could not be opened. Operating system error code +12007(failed to retrieve text for this error. Reason: 15105). +``` + +Operating system error 12007 is a name-resolution failure, not a permission or syntax error: the +statement got past parsing, past credential resolution, and reached the network layer trying to +resolve a host, on both `BULK INSERT` and `OPENROWSET(BULK ...)`. This is the strongest evidence +that the `DATA_SOURCE` form of both statements is genuinely wired to Blob Storage on this +platform, and that a syntactically correct load against a real, reachable storage account with a +valid credential is the expected success path. That last step, an actual load against a real +storage account, was not run in this session: no Azure Blob Storage account was available to the +authoring environment. Everything up to and including the network attempt was verified; a +successful end-to-end transfer was not. + +## WITH options do not change the outcome + +```sql +BULK INSERT dbo.t1 FROM '/tmp/data.csv' + WITH (FIELDTERMINATOR=',', ROWTERMINATOR='\n', TABLOCK, BATCHSIZE=1000); +``` + +still fails with the same Msg 12713. Neither `TABLOCK` nor `BATCHSIZE` bypass the local-path +refusal; they are irrelevant to it. Do not suggest either as a workaround for the error. + +## bcp: present, but its certificate-trust flags failed silently + +`bcp` ships in the image and runs (`bcp -v` prints its banner, exit 0). Getting it to actually +connect to this engine was not achieved in this session, and the failure mode is worth recording +because it is silent in a way that wastes an agent's time. + +The engine presents a self-signed certificate, the same one `sqlcmd` accepts via its own `-C` +flag. Against `bcp`, three different ways of asking the client to trust that certificate were +tried: + +- `-u` (trust server certificate) +- `-Yo` (optional encryption) +- an ODBC DSN (`ODBCINI` pointed at a user-writable file) carrying `TrustServerCertificate=yes` + +All three produced **zero bytes of output on stdout and stderr, and exit code 1**. No SQLState, no +native error, no message at all, confirmed by piping through `xxd`. + +By contrast, the paths that do **not** ask the client to trust the certificate produced real, +diagnosable errors: + +- no `-Y` flag at all: `SSL Provider: [error:0A000086:SSL routines::certificate verify + failed:self-signed certificate]` +- `-Ys` (encryption mandatory, no `-u`): `TCP Provider: Error code 0x2746` + +So the two informative failures are the ones where the client refuses the certificate, and the +three silent failures are every attempt tried to make the client accept it. This looks like a +defect in this build's certificate-trust handling rather than a documented behaviour, and it +should not be presented to a user as expected. What follows from it for this skill: **`bcp`'s +data-transfer behaviour itself, batch size handling, row counts, native versus character mode, +against this engine was not verified**, because no attempt reached a connected state. Treat any +`bcp` guidance in the skill body as based on the tool's documented flags and on how `bcp` behaves +against SQL Server generally, not as measured against this platform. + +## Recovery model: accepted here, and beside the point + +`ALTER DATABASE dq_bulkload SET RECOVERY BULK_LOGGED` was accepted with no error on this engine, +and `sys.databases.recovery_model_desc` reported `BULK_LOGGED` afterward, measured 2026-08-29 on +the local container only. Whether the cloud service accepts or silently ignores the same statement +was not checked and is not claimed either way, because it does not matter: the `BULK INSERT` page's +"Log behavior" section states flatly that "Minimal logging isn't supported in Azure SQL Database" +(fetched 2026-09-03), and the log rate governor below applies regardless of recovery model. That is +why this skill builds no advice on recovery model at all. + +## The transaction log rate governor + +Verified against Microsoft Learn, "Resource Management - Azure SQL Database" +(`resource-limits-logical-server`, section "Transaction log rate governance"), fetched +2026-08-29. Quoting the operative sentence: "Transaction log rate governance is a process in Azure +SQL Database used to limit high ingestion rates for workloads such as bulk insert, SELECT INTO, +and index builds. These limits are tracked and enforced at the subsecond level to the rate of log +record generation, limiting throughput regardless of how many IOs can be issued against data +files." + +The wait types it documents, from `sys.dm_exec_requests` and `sys.dm_os_wait_stats`: + +| Wait type | Meaning | +|---|---| +| `LOG_RATE_GOVERNOR` | Database-level limiting | +| `POOL_LOG_RATE_GOVERNOR` | Elastic pool-level limiting | +| `INSTANCE_LOG_RATE_GOVERNOR` | Instance-level limiting | +| `HADR_THROTTLE_LOG_RATE_SEND_RECV_QUEUE_SIZE` | Replica feedback, Premium/Business Critical replication falling behind | +| `HADR_THROTTLE_LOG_RATE_LOG_SIZE` | Feedback control avoiding an out-of-log-space condition | +| `HADR_THROTTLE_LOG_RATE_MISMATCHED_SLO` | Geo-replication feedback, avoiding secondary unavailability | + +The cap itself is readable per database. `sys.dm_user_db_resource_governance` documents +`primary_max_log_rate` as "Maximum log rate in bytes per second at user workload group level", and +`pool_max_log_rate` and `instance_max_log_rate` as the pool and instance equivalents. Learn's own +example query selects `database_name, primary_group_id, primary_max_log_rate, primary_group_max_io, +pool_max_io` from that view, returning one row for a single database and one row per database in an +elastic pool. Reading the view needs `VIEW DATABASE STATE`, or membership in +`##MS_ServerStateReader##` on Basic, S0, S1 and pooled databases. Fetched 2026-09-03, not measured. + +The same page's mitigation list: scale up to a higher service level or a different tier (Hyperscale +publishes an explicit per-database log rate, 150 MiB/s on premium-series hardware, 100 MiB/s on +other hardware); load transient staging data into `tempdb`, which is minimally logged there; for +analytic loads, target a table with a clustered columnstore index or data compression to reduce the +log volume the load generates. None of this was independently measured against a running load in +this session, since the container used has one fixed compute size and no elastic pool to compare +against; it is carried here from the Learn page directly and dated as such. + +## Platform limits and permissions from Microsoft Learn + +Fetched 2026-09-03 from the `BULK INSERT (Transact-SQL)` page and not independently measured. These +are the parts that differ from SQL Server and are therefore the parts an agent gets wrong. + +| Aspect | Azure SQL Database | +|---|---| +| Data source | Azure Storage only. No local path, no UNC path | +| Source authentication | Microsoft Entra ID, SAS token, or managed identity | +| Unsupported options | `*` wildcards in the path, `FORMAT = 'PARQUET'` | +| Permissions on the target | `INSERT` and `ADMINISTER DATABASE BULK OPERATIONS`, plus `ALTER TABLE` when constraints, triggers or `KEEPIDENTITY` are involved | +| Minimal logging | Not supported | + +Three defaults that make a bad load look like a good one: + +- `MAXERRORS` defaults to 10. Rows that fail conversion are skipped and counted, and the statement + still succeeds. `bcp`'s `-m` has the same default. +- `ERRORFILE` on Azure SQL Database should be accompanied by `ERRORFILE_DATA_SOURCE`, or the import + "might fail with permissions error". The named file must not already exist in the container. +- `FIRSTROW` is 1-based and, in Learn's words, "isn't intended to skip column headers. The + `BULK INSERT` statement doesn't support skipping headers." Skipped rows are scanned for field + terminators only, not validated. `bcp`'s `-F` is the flag that does skip a header row. + +A format file caps out at 1,024 fields, and exceeding it raises error 4822. `bcp` has no such +limit. diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/driver-bulk-apis.md b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/driver-bulk-apis.md new file mode 100644 index 0000000..6348925 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/references/driver-bulk-apis.md @@ -0,0 +1,76 @@ +# The client library bulk paths: documented, not measured here + +SKILL.md carries a runnable snippet for each of the three. This file is what those snippets leave +out. None of it was exercised against a live database in this authoring session: there was no .NET, +Python or Node.js runtime alongside the engine used for +[bulk-load-errors-and-log-rate-governor.md](bulk-load-errors-and-log-rate-governor.md). Treat every +signature here as documented shape, and fetch the linked page before writing code against it. + +## Contents + +- [.NET: SqlBulkCopy](#net-sqlbulkcopy) +- [Python: pyodbc fast_executemany](#python-pyodbc-fast_executemany) +- [Node.js: the mssql package](#nodejs-the-mssql-package) + +## .NET: SqlBulkCopy + +`Microsoft.Data.SqlClient.SqlBulkCopy` takes a `DataTable`, a `DbDataReader` or an `IDataReader` +and streams it to `DestinationTableName`. Beyond the properties in SKILL.md: + +- The behaviour flags are constructor-time, not properties: `SqlBulkCopyOptions` is a bitwise enum + with `KeepIdentity` (1), `CheckConstraints` (2), `TableLock` (4), `KeepNulls` (8), + `FireTriggers` (16), `UseInternalTransaction` (32), `AllowEncryptedValueModifications` (64) and + `CacheMetadata` (128). Defaults are the inverse of `BULK INSERT`'s in the same places: + constraints are not checked, triggers do not fire, row locks rather than a table lock. +- `TableLock` is the `TABLOCK` equivalent and does nothing about the log rate governor either. +- `UseInternalTransaction` puts each batch in its own transaction, so an error rolls back the + current batch and leaves the earlier ones committed. Without it, and without an ambient + transaction, a single bulk copy is one non-transacted operation with nothing to roll back. +- Learn warns that mismatched source and destination types are converted per value, which "can + affect performance, and also can result in unexpected errors". Match the types. +- `CacheMetadata` skips the metadata discovery query on repeat loads to the same table, and Learn + warns it can corrupt data if the schema changes underneath it. Call `ClearCachedMetadata()` after + a schema change or a `ChangeDatabase`. + +Reference: [SqlBulkCopy Class](https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlbulkcopy) +and [SqlBulkCopyOptions Enum](https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlbulkcopyoptions). + +## Python: pyodbc fast_executemany + +`cursor.fast_executemany` is a cursor property, default `False`. pyodbc's own type stub describes +the two modes: false means `executemany()` "does nothing more than iterate over the provided list +of parameters and calls execute() on each set of parameters. This is typically slow"; true means +"the parameters are sent to the database in one bundle (with the SQL)". + +The reason SKILL.md chunks the rows by hand is in the implementation, not in the docs. `ExecuteMulti` +allocates one contiguous buffer sized for every remaining row and sets the ODBC `PARAMSET_SIZE` +attribute to the full converted row count. There is no chunking, no per-row status tracking and no +configurable batch size, so the peak memory of a `fast_executemany` call scales with the whole list +handed to it, and a conversion failure part way through is reported against the batch rather than a +row. + +This is a parameterised `INSERT` bundled efficiently, not the bulk-copy wire protocol that `bcp` +and `SqlBulkCopy` use. It is the fastest route pyodbc offers and it is still generating a log record +per row, so the log rate governor is the ceiling here as everywhere else. + +Which driver to install and how to connect is `connect-from-python`. + +Reference: [pyodbc, Cursor.fast_executemany](https://github.com/mkleehammer/pyodbc/wiki/Cursor#fast_executemany). + +## Node.js: the mssql package + +`request.bulk(table, [options], [callback])` takes a `sql.Table`, and returns a promise resolving to +the row count when no callback is passed. The package's README is explicit that when defining +columns it is critical to state whether each one is nullable, which is why SKILL.md's snippet passes +`{ nullable: ... }` on every column rather than only where it looks necessary. + +`table.create = true` lets the module create the destination table if it does not exist, including +single and multi column primary keys. Leave it `false` when loading into a table someone else owns: +a typo in a column type then fails loudly instead of silently creating a second wrong table. + +Values in `rows.add(...)` are positional, in the order the columns were declared, so a reordered +column list silently loads into the wrong columns. Type mismatches surface as a conversion error +from the driver rather than from the engine, which is why the message will not look like a SQL +error. Connecting in the first place, including Entra ID, is `connect-from-typescript-and-node`. + +Reference: [node-mssql, Bulk load](https://github.com/tediousjs/node-mssql#bulk-load). diff --git a/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/skill.spec.jsonc b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/skill.spec.jsonc new file mode 100644 index 0000000..c4485aa --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/bulk-load-and-bulk-copy/skill.spec.jsonc @@ -0,0 +1,142 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "bulk-load-and-bulk-copy", + "domain": "data-movement", + "value": ["corrects-a-wrong-assumption"], + "correction": "Agents write BULK INSERT, OPENROWSET(BULK ...) and bcp against a local file path exactly as they would on SQL Server, then reach for the SQL Server minimal-logging playbook, TABLOCK, a BULK_LOGGED recovery model, large batches, to make a slow load fast, when in fact Azure SQL Database's server-side bulk load path only ever reads from Blob Storage, a local path is refused with Msg 12713 before the file is even checked for existence, and every load, minimally logged or not, is still capped by a fixed per-service-tier transaction log generation rate enforced by the log rate governor, which throttles ingestion no matter how the load is written.", + "value_probe": { + "gate": "The answer stages the file where the service can actually read it and loads from there, or drives the load from a client that streams the rows over the connection, and treats the throughput question as a service tier limit rather than as hint tuning.", + "task": "Our nightly job loads a 40 million row CSV into a staging table. On the old SQL Server box it ran like this and finished in about eleven minutes:\n\n```sql\nALTER DATABASE staging SET RECOVERY BULK_LOGGED;\nBULK INSERT dbo.stg_orders\nFROM 'D:\\feeds\\orders_20260901.csv'\nWITH (FORMAT = 'CSV', FIRSTROW = 2, TABLOCK, BATCHSIZE = 500000);\n```\n\nWe have just moved the database to Azure SQL Database and the same job does not work at all. The file is produced by an upstream system onto a file share that the job server mounts. Management wants the load back inside the same overnight window it used to fit in. Give us the load path we should be using and how to get the throughput back.", + "states": [ + "The server side bulk load path on this service reads only from Azure Blob Storage, so the file has to be staged there before the engine can read it.", + "The path is rejected before the file is looked for, so this is not a permissions, share mount or missing file problem.", + "The bulk logged recovery model is not available on this service, so that half of the old recipe cannot be reproduced.", + "Log generation is capped per service tier by a governor, so ingestion rate is bounded by the tier no matter how the load statement is written." + ], + "refutes": [ + "The statement will read from the local or mapped path once the service account permissions, the share mount or a credential are corrected.", + "Setting the recovery model to bulk logged and keeping TABLOCK will restore minimal logging and the old throughput on this service.", + "Throughput here is governed mainly by how the load statement is written, so a large enough batch size with the right hints will match the old server.", + "Copying the file to the machine the database runs on, or to a path visible to the engine, is a workable option on this service." + ] + }, + "posture": ["write", "provision"], + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-from-python", + "connect-from-dotnet", + "connect-from-typescript-and-node", + "design-azure-sql-schema", + "sqlpackage-import-export", + "diagnose-resource-pressure", + "restore-and-recover", + "skill-feedback" + ], + "validation": { + "target": "container", + "setup": [ + "a database reachable with an administrative connection", + "a small CSV file and a target table whose columns match it", + "for the Blob paths, an Azure Blob Storage container and either a database scoped credential built from a shared access signature or a data source the loader can reach anonymously" + ], + "assert": [ + "BULK INSERT and OPENROWSET(BULK ...) against a path on the local filesystem both fail with Msg 12713, and the same message appears whether or not the named file exists", + "no WITH option on BULK INSERT, including TABLOCK and BATCHSIZE, changes the Msg 12713 outcome for a local path", + "an ad hoc https Blob URL with no matching CREDENTIAL object fails with Msg 15151 naming the URL itself as the credential it could not find", + "CREATE EXTERNAL DATA SOURCE with TYPE = BLOB_STORAGE is accepted with no credential attached, and a BULK INSERT through it reaches the network layer rather than being rejected outright", + "ADMINISTER DATABASE BULK OPERATIONS is a permission this engine recognises at database scope, which is the Azure SQL Database spelling of SQL Server's ADMINISTER BULK OPERATIONS", + "sys.dm_user_db_resource_governance exposes primary_max_log_rate, so the log rate cap the skill tells the reader to check is readable from inside the database", + "bcp is present in the engine container image, not on the host, and its version banner runs without a live connection" + ], + "probes": [ + { + "sql": "SELECT * FROM OPENROWSET(BULK '/tmp/probe-bulk-load.csv', SINGLE_CLOB) AS x;", + "expect": { + "error": 12713 + }, + "why": "The local filesystem path is the habit this skill exists to break, and Msg 12713 is the refusal that proves bulk load on this service reads Blob Storage and nothing else. Measured 2026-08-29 on an engine reporting EngineEdition 5. If this ever stops failing, the correction at the top of the skill is stale." + }, + { + "sql": "SELECT * FROM OPENROWSET(BULK '/tmp/probe-never-created.csv', SINGLE_CLOB) AS x;", + "expect": { + "error": 12713 + }, + "why": "The same refusal for a path that was never created. This is the probe that separates a platform rule from a missing file: without it, the first probe is equally consistent with the engine opening the file and failing, which is exactly the wrong conclusion an agent draws from the message text alone." + }, + { + "sql": "SELECT * FROM OPENROWSET(BULK 'https://example.blob.core.windows.net/probe-container/data.csv', SINGLE_CLOB) AS x;", + "expect": { + "error": 15151 + }, + "why": "An ad hoc Blob URL with no credential object of that exact name fails with Msg 15151, which names the whole URL as the credential it could not find and therefore reads like an access denial when it is not one. Measured 2026-08-29. Pinning the number is what lets the skill keep saying it means no such credential object exists." + }, + { + "sql": "CREATE TABLE dbo.probe_bulk_target (c1 nvarchar(100) NULL);\nBULK INSERT dbo.probe_bulk_target FROM '/tmp/probe-bulk-load.csv'\n WITH (FIELDTERMINATOR = ',', ROWTERMINATOR = '0x0a', TABLOCK, BATCHSIZE = 1000);", + "expect": { + "error": 12713 + }, + "why": "TABLOCK and BATCHSIZE are the two options an agent reaches for the moment a load is called slow, and neither one moves the local-path refusal. Measured 2026-08-29: the identical Msg 12713 with both options present. This is the probe that keeps the corresponding do-not rule honest rather than merely asserted." + }, + { + "sql": "CREATE EXTERNAL DATA SOURCE probe_blob\n WITH (TYPE = BLOB_STORAGE, LOCATION = 'https://example.blob.core.windows.net/probe-container');", + "expect": { + "succeeds": true + }, + "why": "The DATA_SOURCE form is the only legal bulk-load route on this service, so the skill's headline statement is worthless if this DDL is not accepted. Measured 2026-08-29: accepted with no error and with no CREDENTIAL attached, which is also what makes the anonymous-container case in the body real. A failure here means the shape in SKILL.md cannot be pasted, and the skill has to be rewritten rather than retuned." + }, + { + "sql": "SELECT CAST(HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'ADMINISTER DATABASE BULK OPERATIONS') AS int) AS has_bulk_permission;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "Microsoft Learn's BULK INSERT permissions section, fetched 2026-09-03, says Azure SQL Database requires INSERT and ADMINISTER DATABASE BULK OPERATIONS where SQL Server requires ADMINISTER BULK OPERATIONS. HAS_PERMS_BY_NAME returns NULL for a permission name that is not valid at the named scope, so a NULL here would mean the skill is quoting the SQL Server spelling and the permission bullet is wrong. A 0 would mean the name is right but the probe connection is not administrative." + }, + { + "sql": "SELECT primary_max_log_rate FROM sys.dm_user_db_resource_governance;", + "expect": { + "error": 208 + }, + "why": "This probe exists to pin a container versus cloud difference that cost this skill a wrong instruction. Learn documents sys.dm_user_db_resource_governance and its primary_max_log_rate column for Azure SQL Database, and an earlier revision told the reader to run exactly this to read their tier's log rate ceiling. Measured 2026-09-03 on the container: Msg 208, Invalid object name. The view is not there. sys.dm_db_resource_stats is absent too, and sys.dm_internal_resource_governor_resource_pools does exist but returns zero rows, so the log rate governor is not observable locally at all. The instruction is correct in the cloud and impossible on the container, and the body now says so. If this probe ever starts succeeding, the container gained the cloud DMV and the body needs revisiting." + }, + { + "cmd": "/opt/mssql-tools18/bin/bcp -v", + "where": "container", + "expect": { + "succeeds": true + }, + "why": "The skill routes a whole class of loads to bcp and states that bcp ships inside the engine image rather than on the host, so where this probe runs is half the claim. Measured 2026-08-29, exit 0, version 18.6.0002.1. Run on the host it fails with command not found, which is the finding, not a bug. It says nothing about bcp reaching a connected state, which was never achieved against this engine." + } + ] + }, + "triggering": { + "explicit": [ + "Load this CSV into my Azure SQL Database table as fast as possible", + "Write a BULK INSERT statement to import data from blob storage", + "How do I use bcp to bulk copy data into Azure SQL Database", + "Use SqlBulkCopy to load a DataTable into Azure SQL Database from .NET", + "What is the fastest way to bulk insert rows from Python into Azure SQL Database" + ], + "implicit": [ + "My BULK INSERT script works fine on SQL Server but fails here with an OPENROWSET error", + "Loading a few million rows is taking forever and I don't know what's throttling it", + "I need to get a large CSV from my laptop into the database without writing a loop of INSERT statements", + "Our nightly load started hitting waits it never used to have once the row count grew", + "How do I get OPENROWSET to read a file for a bulk load" + ], + "routing": [ + "load a few hundred thousand rows into Azure SQL from Python" + ], + "negative": [ + "Design a schema for this CSV before I load anything into it", + "Why does my Python app fail to connect with a driver not found error", + "Export my whole database schema and data to a BACPAC for a staging copy", + "Walk me through a zero downtime column rename for this table", + "Why is this single SELECT query slow" + ] + } +} diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export deleted file mode 120000 index 538e9e7..0000000 --- a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export +++ /dev/null @@ -1 +0,0 @@ -../../../skills/sqlpackage-import-export \ No newline at end of file diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/SKILL.md b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/SKILL.md new file mode 100644 index 0000000..4507808 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/SKILL.md @@ -0,0 +1,249 @@ +--- +name: sqlpackage-import-export +description: >- + Moves a whole Azure SQL Database as a portable file with SqlPackage, choosing between the + Extract, Publish, Export and Import actions, stating what each one carries, and giving the + command line for each. Use when asked to export a database to a bacpac, extract or publish a + dacpac, clone or move a database between servers, move the database itself from the local Azure + SQL Database container up to Azure SQL Database, explain dacpac versus bacpac, or diagnose a + failed sqlpackage run such as SQL71627, or SQL71659 when an Import stops because the target + database is not empty. Making the application code run unchanged in the cloud is + azuresql-db-local-to-cloud, and a first bacpac into the local container is azuresql-db-import. +--- + +# Move a whole database with SqlPackage: dacpac and bacpac + +Verified 2026-09-02 against SqlPackage 170.4.83.3, and the engine behaviour below measured +2026-08-29 against a container reporting `EngineEdition` 5, `12.0.2000.8`. + +This owns the four-action decision for moving an entire existing database as a portable file, and +the command line for each action. Building a dacpac from a source-controlled SQL project, shipping +it through CI, its refactorlog and its pre and post deployment scripts are `sql-database-projects`. +Azure SQL Database's own automated backups and point-in-time or geo-restore are +`restore-and-recover`. Loading rows into a table that already exists is `bulk-load-and-bulk-copy`. +Vector indexes themselves are `azuresql-db-rag` and `vector-search-azure-sql`. + +## Read the property list for the action before you write the command + +Properties are per action. `ExtractAllTableData` is an Extract property and is not accepted by +Export. There is no catalog of properties that spans the actions, so ask the build in front of you: + +```bash +sqlpackage /version:True +sqlpackage /Action:Export /? +``` + +An unknown property is refused before any connection is attempted: + +```text +$ sqlpackage /Action:Export /p:ExportAllTableData=true /TargetFile:app.bacpac /SourceServerName:your-server.database.windows.net /SourceDatabaseName:app +*** 'ExportAllTableData' is not a valid argument for the 'Export' action. +``` + +A property can also be real, documented and still unavailable: `/p:Storage=File` is the .NET +Framework build's default, and Learn records `Memory` as the only option on the cross-platform +build. + +Every parameter in this skill is written in long form. The same help output gives each one a short +form, `/Action` as `/a`, `/SourceFile` as `/sf`, `/TargetFile` as `/tf`, and the two forms are +interchangeable. + +## The four actions, as four commands + +| Action | Direction | Default contents | File | +|---|---|---|---| +| Extract | database to file | schema only | writes `.dacpac` | +| Publish | file to database | schema only | reads `.dacpac` | +| Export | database to file | schema and every base table's rows | writes `.bacpac` | +| Import | file to database | schema and every base table's rows | reads `.bacpac` | + +Extract: + +```bash +sqlpackage /Action:Extract /TargetFile:app.dacpac /DiagnosticsFile:extract.log \ + /SourceServerName:your-server.database.windows.net /SourceDatabaseName: \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" \ + /p:ExtractAllTableData=false /p:VerifyExtraction=true +``` + +Export: + +```bash +sqlpackage /Action:Export /TargetFile:app.bacpac /DiagnosticsFile:export.log \ + /SourceServerName:your-server.database.windows.net /SourceDatabaseName: \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" \ + /p:VerifyExtraction=true /p:LongRunningCommandTimeout=0 +``` + +Import: + +```bash +sqlpackage /Action:Import /SourceFile:app.bacpac /DiagnosticsFile:import.log \ + /TargetServerName:your-server.database.windows.net /TargetDatabaseName: \ + /TargetUser: /TargetPassword:"$SQLPACKAGE_PASSWORD" \ + /p:DatabaseEdition=Standard /p:DatabaseServiceObjective=S1 +``` + +Publish: + +```bash +sqlpackage /Action:Publish /SourceFile:app.dacpac \ + /TargetServerName:your-server.database.windows.net /TargetDatabaseName: \ + /TargetUser: /TargetPassword:"$SQLPACKAGE_PASSWORD" \ + /p:BlockOnPossibleDataLoss=true /p:DropObjectsNotInSource=false +``` + +`/p:DatabaseEdition` and `/p:DatabaseServiceObjective` are accepted by Import and Publish and are +how the database they create is sized. Extract and Export do not accept them. + +## Both Import and Publish can create the target; only Publish can rerun + +Microsoft Learn states it for each. Publish: if the database doesn't exist on the server, the +publish operation creates it, otherwise an existing database is updated. Import: a new database +can be created by the import action when the authenticated user has create database permissions. + +What Import will not do is load into a target that already holds a user object: + +```text +Error SQL71659: Data cannot be imported into target because it contains one or more user objects. +``` + +It refuses before any statement runs against the target, and retrying the same Import against the +same name refuses again, every time, until that database is dropped and recreated or a new name is +chosen. Publish diffs instead: rerun it against a target it already published to and it reports +`Update complete` with nothing to apply. + +## The property decides what is in the file, not the extension + +An Extract of a database with one table and one view produced a `.dacpac` carrying no data entry +of any kind, and an Export of the same database produced a `.bacpac` carrying +`Data/dbo.Widget/TableData-000-00000.BCP`, one entry per base table holding rows. + +Then the same Extract with one property added: + +```bash +sqlpackage /Action:Extract /TargetFile:app-with-data.dacpac \ + /SourceServerName:your-server.database.windows.net /SourceDatabaseName: \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" /p:ExtractAllTableData=true +``` + +That file is still named `.dacpac`, is still the output of Extract, and now carries +`Data/dbo.Widget/TableData-000-00000.BCP` as well. `/p:TableData` is the narrower form, accepted +by both Extract and Export, naming one table per occurrence: + +```bash +sqlpackage /Action:Extract /TargetFile:app-two-tables.dacpac \ + /SourceServerName:your-server.database.windows.net /SourceDatabaseName: \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" \ + /p:TableData=dbo.Widget /p:TableData=dbo.WidgetAudit +``` + +Open the file and look rather than reading the name: + +```bash +unzip -l app-with-data.dacpac +``` + +## An export is not a backup, and is not consistent by default + +Microsoft Learn states plainly that bacpac files aren't intended to be used for backup and restore +operations. `BACKUP DATABASE app TO DISK='/tmp/x.bak'` against this engine returns `Msg 40510`, and +there is no RESTORE either. + +Learn again, on Export: for an export to be transactionally consistent, either no write activity is +occurring during the export, or the export is taken from a transactionally consistent copy of the +database. Nothing in the output of a successful export says which of those was true. Export from a +database copy, or from a database nothing is writing to, or state in the answer that the file may +be internally inconsistent. + +## Before an Export or Extract from a database with operating history + +Export is limited to the Azure SQL Database surface area, so a source carrying an element outside +it fails with `SQL71627` after the whole schema model has already been built, discarding the entire +attempt rather than the one object. The usual carriers are Windows-authenticated users and logins, +such as an inherited `NT AUTHORITY\SYSTEM`, and leftover Service Broker or query notification +permissions, most often a `RECEIVE` grant on `QueryNotificationErrorsQueue`. + +Open [references/preflight-and-sql71627.md](references/preflight-and-sql71627.md) before running an +export against a database you did not create yourself: it holds the queries that find both classes +and the remediation for each. + +Extract accepts two properties that drop the offending model elements, and Export accepts neither: + +```bash +sqlpackage /Action:Extract /TargetFile:app.dacpac \ + /SourceServerName:your-server.database.windows.net /SourceDatabaseName: \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" \ + /p:IgnoreUserLoginMappings=true /p:IgnorePermissions=true +``` + +So when a database will not export, extracting the schema and moving the rows separately is the +route that still runs. + +## Read the plan before a Publish that could drop something + +DeployReport writes XML and Script writes T-SQL. Neither one changes the target database. + +```bash +sqlpackage /Action:DeployReport /SourceFile:app.dacpac /DeployReportPath:deploy-report.xml \ + /TargetServerName:your-server.database.windows.net /TargetDatabaseName: \ + /TargetUser: /TargetPassword:"$SQLPACKAGE_PASSWORD" +``` + +```bash +sqlpackage /Action:Script /SourceFile:app.dacpac /DeployScriptPath:publish.sql \ + /TargetServerName:your-server.database.windows.net /TargetDatabaseName: \ + /TargetUser: /TargetPassword:"$SQLPACKAGE_PASSWORD" +``` + +`/p:BlockOnPossibleDataLoss` defaults to True and is what stops a publish that would drop data. +Leave it on. `/p:CreateNewDatabase=true` does not mean "create it if it is missing", which Publish +already does: Learn describes it as whether the target database should be updated or whether it +should be dropped and re-created. Setting it against a populated target destroys that target. + +## A vector index breaks the round trip, and the export looks fine + +Check before planning anything around a bacpac: + +```sql +SELECT OBJECT_NAME(object_id) AS table_name, name +FROM sys.indexes WHERE type_desc = 'VECTOR'; +``` + +An import creates schema objects before it loads data, so a vector index is created against an +empty table. Measured 2026-08-31 against `12.0.2000.8`: a vector index needs at least 100 rows +carrying non-null vectors, 99 is refused with `Msg 42266` and 100 succeeds. Drop the vector indexes +before exporting and recreate them after the data is loaded. `TRUNCATE TABLE` is refused while a +vector index exists (`Msg 42232`), so reloading in place hits the same wall one step earlier. + +## Check it worked + +Exit code 0 says the action SqlPackage attempted finished. It does not say the rows arrived. + +```bash +unzip -l app.bacpac | grep TableData +``` + +Expect one `Data/./TableData-000-00000.BCP` entry per base table that held rows, and +none for a view. Then compare the target against the source: + +```sql +SELECT SUM(p.rows) AS row_count, COUNT(DISTINCT t.object_id) AS table_count +FROM sys.tables AS t +JOIN sys.partitions AS p ON p.object_id = t.object_id AND p.index_id IN (0, 1); +``` + +Run it on both databases. Equal numbers, or a stated reason they differ, is the check. A schema +only dacpac publishes to zero rows and that is correct, not a failure. + +## References + +- [Measured runs, file listings, exit codes and provenance](references/dacpac-bacpac-contents-and-errors.md): open it when a claim above disagrees with what you are seeing, or to reproduce a measurement. +- [The SQL71627 pre-flight scan](references/preflight-and-sql71627.md) +- The property reference per action, and the only authority on whether a property exists: + [Extract](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-extract), + [Export](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-export), + [Import](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-import), + [Publish](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-publish), + [Script](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-script) and + [DeployReport](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-deploy-drift-report). diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/dacpac-bacpac-contents-and-errors.md b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/dacpac-bacpac-contents-and-errors.md new file mode 100644 index 0000000..bc6f390 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/dacpac-bacpac-contents-and-errors.md @@ -0,0 +1,181 @@ +# What lands inside a dacpac and a bacpac, and the errors on the way + +## Contents + +- [Provenance](#provenance) +- [Setup](#setup) +- [Extract versus Export: what lands inside the file](#extract-versus-export-what-lands-inside-the-file) +- [ExtractAllTableData: a dacpac that carries data](#extractalltabledata-a-dacpac-that-carries-data) +- [Import into a new, empty target](#import-into-a-new-empty-target) +- [Publish creates the target database](#publish-creates-the-target-database) +- [Import refuses a non-empty target](#import-refuses-a-non-empty-target) +- [Publish reruns against its own target with no changes](#publish-reruns-against-its-own-target-with-no-changes) +- [There is no BACKUP T-SQL](#there-is-no-backup-t-sql) +- [Environment](#environment) + +## Provenance + +Every parameter and `/p:` property named in `SKILL.md` was read out of +`sqlpackage /Action: /?` on SqlPackage 170.4.83.3 on 2026-09-02, then cross-checked against +the Microsoft Learn page for that same action. Nothing is stated in the skill that failed both +checks, and nothing is carried from memory. + +Engine behaviour was measured on 2026-08-29 against the Azure SQL Database container recorded under +[Environment](#environment), and the vector index row limits on 2026-08-31 against the same build. +Everything below is a transcript of those runs, not a reconstruction. + +## Setup + +A source database with one table holding three rows and one view over it: + +```sql +CREATE TABLE dbo.Widget (Id INT IDENTITY PRIMARY KEY, Name NVARCHAR(100)); +INSERT INTO dbo.Widget (Name) VALUES ('alpha'),('beta'),('gamma'); +GO +CREATE VIEW dbo.WidgetView AS SELECT Id, Name FROM dbo.Widget; +``` + +## Extract versus Export: what lands inside the file + +```text +$ sqlpackage /Action:Extract /SourceConnectionString:"..." /TargetFile:widget.dacpac +Successfully extracted database and saved it to file '.../widget.dacpac'. +Time elapsed 0:00:07.56 + +$ sqlpackage /Action:Export /SourceConnectionString:"..." /TargetFile:widget.bacpac +Processing Table '[dbo].[Widget]'. +Successfully exported database and saved it to file '.../widget.bacpac'. +Time elapsed 0:00:07.69 +``` + +```text +$ unzip -l widget.dacpac + 5067 model.xml + 189 DacMetadata.xml + 175 [Content_Types].xml + 1643 Origin.xml +--------- + 7074 4 files + +$ unzip -l widget.bacpac + 5957 model.xml + 193 DacMetadata.xml + 46 Data/dbo.Widget/TableData-000-00000.BCP + 1693 Origin.xml + 357 _rels/.rels + 340 [Content_Types].xml +--------- + 8586 6 files +``` + +The bacpac carries one `Data/` entry, for the base table. `WidgetView` produced no data entry in +either file, because a view has no rows of its own. + +## ExtractAllTableData: a dacpac that carries data + +```text +$ sqlpackage /Action:Extract /SourceConnectionString:"..." /TargetFile:widget_withdata.dacpac \ + /p:ExtractAllTableData=true +Processing Table '[dbo].[Widget]'. +Successfully extracted database and saved it to file '.../widget_withdata.dacpac'. + +$ unzip -l widget_withdata.dacpac + 5067 model.xml + 189 DacMetadata.xml + 46 Data/dbo.Widget/TableData-000-00000.BCP + 1643 Origin.xml + 357 _rels/.rels + 340 [Content_Types].xml +--------- + 7642 6 files +``` + +Same action, same `.dacpac` extension, and now the same `Data/` shape a bacpac carries. Nothing +about the filename or the action name changes; only the property does. + +## Import into a new, empty target + +```text +$ sqlpackage /Action:Import /SourceFile:widget.bacpac /TargetConnectionString:"...dq_from_bacpac..." +Importing data +Processing Table '[dbo].[Widget]'. +Successfully imported database. +Time elapsed 0:00:39.17 +``` + +```sql +SELECT COUNT(*) FROM dq_from_bacpac.dbo.Widget; -- 3 +``` + +## Publish creates the target database + +```text +$ sqlpackage /Action:Publish /SourceFile:widget.dacpac /TargetConnectionString:"...dq_from_dacpac..." +Updating database (Start) +Creating database dq_from_dacpac... +Creating Table [dbo].[Widget]... +Creating View [dbo].[WidgetView]... +Update complete. +Successfully published database. +``` + +The target database `dq_from_dacpac` did not exist before this run; confirmed with +`SELECT name FROM sys.databases` before and after. Row count in the published target is 0, +because the source file was a schema-only dacpac: + +```sql +SELECT COUNT(*) FROM dq_from_dacpac.dbo.Widget; -- 0 +``` + +## Import refuses a non-empty target + +Running Import a second time against `dq_from_bacpac`, which the first Import had already +populated: + +```text +$ sqlpackage /Action:Import /SourceFile:widget.bacpac /TargetConnectionString:"...dq_from_bacpac..." +Creating deployment plan +Initializing deployment +*** Error importing database: Data cannot be imported into target because it contains one or more +user objects. Import should be performed against a new, empty database. +Error SQL71659: Data cannot be imported into target because it contains one or more user objects. +``` + +Refused before any statement ran against the target; the row count in `dq_from_bacpac` was +unchanged. + +## Publish reruns against its own target with no changes + +Running the same Publish a second time against `dq_from_dacpac2`, already fully published: + +```text +$ sqlpackage /Action:Publish /SourceFile:widget.dacpac /TargetConnectionString:"...dq_from_dacpac2..." +Initializing deployment (Complete) +Analyzing deployment plan (Complete) +Updating database (Start) +Update complete. +Successfully published database. +Time elapsed 0:00:01.66 +``` + +No object-creation lines the second time; the plan had nothing to apply. Compare the 32-second +first run, which created the database and both objects, against this 1.66-second no-op rerun. + +## There is no BACKUP T-SQL + +```text +$ sqlcmd -Q "BACKUP DATABASE dq_sqlpackage TO DISK='/tmp/x.bak';" +Msg 40510, Level 16, State 1, Server SQL Azure, Line 1 +'BACKUP DATABASE' is not supported in this version of SQL Server. +``` + +## Environment + +SqlPackage 170.4.83.3 installed as a global .NET tool (`dotnet tool install -g +microsoft.sqlpackage`), against a container reporting: + +```sql +SELECT @@VERSION; -- Microsoft SQL Azure (RTM) - 12.0.2000.8 +SELECT SERVERPROPERTY('EngineEdition'); -- 5 +SELECT SERVERPROPERTY('Edition'); -- SQL Azure +``` diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/preflight-and-sql71627.md b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/preflight-and-sql71627.md new file mode 100644 index 0000000..86aa606 --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/references/preflight-and-sql71627.md @@ -0,0 +1,76 @@ +# The pre-flight scan for SQL71627, sourced from documentation + +This file is sourced from Microsoft Learn, Microsoft support content and the SqlPackage issue +tracker, not reproduced against a live engine. The container used to verify the rest of this +skill accepted a Windows-style login-mapped user without complaint, because its permission model +is looser than a real Azure SQL Database logical server on this specific point; the failure below +is real and documented, but it was not observed running locally. Treat the queries as a starting +point, and treat a clean scan on the container as inconclusive rather than as proof the same +export will succeed against a real Azure SQL Database. + +## What SQL71627 means + +`sqlpackage /Action:Export` and `/Action:Extract` build a schema model of the source database +before writing anything, then validate that model against the target platform. SQL71627 is that +validation failing: an element in the model has a property set to a value Azure SQL Database has +never supported. The message names the element and the property, but the failure surfaces only +after the full schema model has already been built, and the whole export is discarded, not just +the offending object. + +## The two most common causes + +**Windows-authenticated users and logins.** A user or login created `FOR LOGIN` against a Windows +principal, or one whose `AuthenticationType` or `IsMappedToWindowsLogin` property is set, has no +equivalent in Azure SQL Database, which supports only SQL authentication and Microsoft Entra +authentication. Accounts such as `NT AUTHORITY\SYSTEM` or `NT AUTHORITY\NETWORK SERVICE`, left +over from an inherited source or a migrated instance, are the usual carriers. + +Find them before exporting: + +```sql +SELECT dp.name AS principal_name, dp.type_desc, sp.name AS login_name +FROM sys.database_principals AS dp +LEFT JOIN sys.server_principals AS sp ON dp.sid = sp.sid +WHERE dp.type IN ('U', 'G') -- Windows user or group principals + OR sp.type IN ('U', 'G'); +``` + +Remediation: drop the user, or recreate it as a contained database user (`CREATE USER ... WITH +PASSWORD = ...` or a Microsoft Entra principal) before exporting again. + +**Leftover Service Broker and query notification permissions.** A database that once used SQL +Server query notification or Service Broker outside Azure SQL Database can carry a `RECEIVE` grant on +`QueryNotificationErrorsQueue` or another Service Broker object. Azure SQL Database's Service +Broker support does not include that surface, and the permission alone is enough to fail the +model. + +Find them before exporting: + +```sql +SELECT grantee_principal_id, permission_name, class_desc, major_id +FROM sys.database_permissions +WHERE permission_name IN ('RECEIVE', 'SEND') + OR major_id IN ( + SELECT object_id FROM sys.objects WHERE name = 'QueryNotificationErrorsQueue' + ); +``` + +Remediation: revoke the permission (`REVOKE RECEIVE ON QueryNotificationErrorsQueue FROM +;`) before exporting again. + +## Reading the error when it happens anyway + +The message names the element type and the unsupported property directly, for example: + +```text +Error SQL71627: The element User has property AuthenticationType set to a value that is not +supported in Microsoft Azure SQL Database v12. +``` + +Search the model for that element type and fix it, rather than re-running the export unchanged +and hoping it was transient. It is not transient; the object is still there. + +## Sources + +- [Error SQL71627 exporting BACPAC file in Azure SQL Database](https://techcommunity.microsoft.com/blog/azuredbsupport/error-sql71627-exporting-bacpac-file-in-azure-sql-database/4266029), Microsoft Tech Community. +- [SqlPackage Export](https://learn.microsoft.com/en-us/sql/tools/sqlpackage/sqlpackage-export) and [SqlPackage Extract](https://learn.microsoft.com/en-us/sql/tools/sqlpackage/sqlpackage-extract), Microsoft Learn, for the full property and permission reference. diff --git a/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/skill.spec.jsonc b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/skill.spec.jsonc new file mode 100644 index 0000000..b5801be --- /dev/null +++ b/plugins/microsoft-sql-migration/skills/sqlpackage-import-export/skill.spec.jsonc @@ -0,0 +1,152 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "sqlpackage-import-export", + "domain": "data-movement", + "value": ["not-in-training-data", "multi-step-with-silent-failures"], + "correction": "Agents treat the dacpac and bacpac extensions as what decides what moves, when what actually decides it is which SqlPackage action ran and which property was set: Extract and Publish carry schema only by default, Export and Import always carry schema plus every base table's row data, and Extract can be told, with ExtractAllTableData, to fold row data into a file still named and shaped like a schema-only dacpac. Agents also read a completed publish or import as proof the database is now backed up, when Azure SQL Database has no BACKUP or RESTORE T-SQL and neither bacpac nor dacpac is a supported recovery mechanism. Agents assume Import behaves like Publish, when Import refuses outright, with error SQL71659, against any target database that already holds a single user object, while Publish diffs an existing target and reruns safely; both actions create the target database when it does not exist and the connected principal has permission to create one, so the difference between them is the refusal, not the creation. Finally, agents export a live database without scanning it first, when a database still carrying Windows-authenticated users, server-login-mapped users or leftover Service Broker permissions fails export with SQL71627 only after schema extraction has already run, discarding the whole attempt rather than the offending object.", + "value_probe": { + "gate": "The answer chooses the action by what actually has to move and by whether the target already holds objects, checks the source for principals the target cannot carry before the long running step, and separates this copy operation from the question of what the database's real backup is.", + "task": "We need a copy of our production Azure SQL Database in a brand new database on our development server, schema and data both, and we want to repeat it every Monday morning. I produced a file from production last week and now I want to lay it down on the development side.\n\nThree things I am unsure about. The development database does not exist yet, so do I have to go and create an empty one first? On the second Monday that database will already have last week's tables in it, so will the same command just put the new copy over the top? And our runbook currently names this weekly file as the backup of record for the application database, which I would like to confirm before the audit next month.\n\nAlso, last week's run against production failed part of the way through, complaining about a user, after it had already been going for twenty minutes. Tell me the sequence to use.", + "states": [ + "What decides whether row data travels is which action was run and which properties were set, not the file's name or extension.", + "The action that always carries data refuses a target that already holds any user object, so the second Monday needs a fresh target or the action that diffs an existing one.", + "The target database is created automatically when it is absent and the connected principal is allowed to create one, so creating an empty database first is not required.", + "Neither of these file types is a supported recovery mechanism for this service, so the runbook should name the service's own automatic backups instead.", + "A principal the target cannot honour fails the export only after schema extraction has already run, discarding the whole attempt, so the source should be scanned for such principals first." + ], + "refutes": [ + "The target database has to exist before the operation runs, because the tool will not create a database that is absent.", + "The file extension is what determines whether row data is included in the file.", + "A schema and data file of this kind is a supported backup and recovery mechanism for an Azure SQL Database.", + "The data carrying import can be re-run against a target that already contains user objects and will overwrite or merge them.", + "The failure naming a user is a transient fault or a permissions problem on the tool's own connection, so retrying will get past it." + ] + }, + "posture": ["read", "write", "execute"], + // REPRIEVED 2026-09-20, and this comment replaces one that is now wrong. The + // reading of 2026-09-05 returned NEITHER and this field was lowered on it, but + // that reading rested on three runs per arm. Re-measured at ten, the verdict is + // LIFT: the statement that a principal the target cannot honour fails the export + // only after schema extraction has already run reads 9 of 10 with the skill + // against 6 of 10 without. A thirty point gap with ten runs behind it is exactly + // what the rule asks for, so the skill does change the answer and the earlier + // reading was the sample size talking. + // + // RAISED TO GA on 2026-09-21. It had stayed at draft only because raising a + // maturity is a judgement nobody had made yet, not because the evidence was + // missing. All ten of its engine checks pass in the live run of 2026-09-20 and + // the value verdict is LIFT at ten runs per arm, which is both rungs, so + // scripts/check-maturity-drift.mjs computes a ga ceiling and this field is now + // at it. The earlier three-run reading that demoted it is superseded and + // nothing should be decided from it again. + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "sql-database-projects", + "restore-and-recover", + "bulk-load-and-bulk-copy", + "provision-azure-sql-db" + ], + "validation": { + "target": "both", + "setup": [ + "a source database, container or cloud, holding at least one table with rows and one view", + "SqlPackage installed as a global .NET tool", + "a second, empty target database to import or publish into", + "a third, non-empty target database to demonstrate the import refusal" + ], + "assert": [ + "sqlpackage /Action:Extract against the source database produces a file whose only entries are model.xml, DacMetadata.xml, Origin.xml and Content_Types.xml, with no Data folder", + "sqlpackage /Action:Export against the same source produces a file that additionally contains a Data/.
/TableData-*.BCP entry for every base table holding rows, and no such entry for a view", + "sqlpackage /Action:Extract with /p:ExtractAllTableData=true produces a .dacpac file that also contains a Data/.
/TableData-*.BCP entry", + "sqlpackage /Action:Import into the non-empty target fails with error SQL71659 and the target is refused before any change is made", + "sqlpackage /Action:Import into the empty target succeeds and the row counts in the target match the source", + "sqlpackage /Action:Publish of the extracted dacpac against an already-published target completes with no pending changes and exit code 0", + "BACKUP DATABASE issued as T-SQL against the source database is rejected with error 40510" + ], + "probes": [ + { + "where": "host", + "cmd": "sqlpackage /version:True", + "expect": { "succeeds": true }, + "why": "Every command in this skill is a SqlPackage command, so the tool being installed and able to print its build number with no database in reach is the precondition for all of them. Measured 2026-09-02, exit 0, build 170.4.83.3." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:Extract /help:True | grep -q '^/p:ExtractAllTableData=(BOOLEAN .False.)$'", + "expect": { "succeeds": true }, + "why": "The property this skill turns on, checked to exist for the Extract action and to default to False, out of the build's own help. Measured 2026-09-02 on 170.4.83.3. If a future build renames it or changes the default, the claim that a dacpac is schema only until this is set stops being true, and this probe goes red first." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:Export /help:True | grep -q '^/p:IgnoreUserLoginMappings=' && exit 1 || exit 0", + "expect": { "succeeds": true }, + "why": "The asymmetry the skill routes on: Extract can be told to drop user-to-login mappings and Export cannot, so a database that fails export can still be extracted. Measured 2026-09-02 on 170.4.83.3, the property is absent from the Export help. This probe is deliberately the negative, because the skill's advice becomes wrong the day Export gains it." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:Extract /help:True | grep -q '^/p:IgnoreUserLoginMappings=(BOOLEAN .False.)$'", + "expect": { "succeeds": true }, + "why": "The positive half of the same pair, so a red run on the previous probe can be told apart from SqlPackage renaming or dropping the property everywhere. Measured 2026-09-02 on 170.4.83.3." + }, + { + "where": "host", + "cmd": "sqlpackage /Action:Export /TargetFile:/tmp/probe-sqlpackage.bacpac /SourceServerName:127.0.0.1,$PROBE_PORT /SourceDatabaseName:$PROBE_DB /p:ExportAllTableData=true 2>&1 | grep -q \"is not a valid argument for the 'Export' action\"", + "expect": { "succeeds": true }, + "why": "An invented property is refused by name before any connection is opened, which is what makes a property claim checkable for free. Measured 2026-09-02 on 170.4.83.3: exit 1 and that message, with no connection line in the output, against a port nothing is listening on. This is the probe shape that gives the catalog a cheap way to fail a made-up flag." + }, + { + "sql": "SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS engine_edition;", + "expect": { "rows": [[5]] }, + "why": "The precondition for the no-BACKUP claim. Msg 40510 for BACKUP DATABASE is a property of EngineEdition 5, so the probe below run against anything else would be measuring a different product. Measured 2026-08-29 against 12.0.2000.8 reporting Edition SQL Azure." + }, + { + "sql": "DECLARE @db sysname = DB_NAME();\nBACKUP DATABASE @db TO DISK = '/tmp/probe-sqlpackage.bak';", + "expect": { "error": 40510 }, + "why": "The reason a bacpac gets reached for as a backup, made executable: there is no BACKUP statement on this service, so a bacpac is the only thing that looks like one and it is not one. Measured 2026-08-29 on the container as BACKUP DATABASE against a literal existing database name, Msg 40510. The variable form is used here only so the probe can name the scratch database it is handed; BACKUP DATABASE { database_name | @database_name_var } is the documented syntax (https://learn.microsoft.com/sql/t-sql/statements/backup-transact-sql). If this ever succeeds, this skill's routing to restore-and-recover needs rewriting." + } + , + { + "sql": "DECLARE @sql nvarchar(max) = N'BACKUP DATABASE ' + QUOTENAME(DB_NAME()) + N' TO DISK = ''/tmp/probe_sqlpkg.bak'';';\nEXEC sp_executesql @sql;", + "expect": { + "error": 40510 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. BACKUP DATABASE is not available on this engine, which is why a data tier package is the export path rather than a backup file. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_sp_99 (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_sp_99 (v) SELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 99);\nGO\nCREATE VECTOR INDEX probe_sp_99_ix ON dbo.probe_sp_99 (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42266 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The 100 row minimum is why a bacpac import fails: the import creates schema objects before it loads data, so the index is built against an empty table. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_sp_tr (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_sp_tr (v) SELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_sp_tr_ix ON dbo.probe_sp_tr (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nTRUNCATE TABLE dbo.probe_sp_tr;", + "expect": { + "error": 42232 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And why reloading in place hits the same wall one step earlier: the truncate that would empty the table is refused while the index exists. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Export my Azure SQL database to a bacpac file", + "How do I copy this database to another server with sqlpackage", + "What is the difference between a dacpac and a bacpac", + "Extract a dacpac from my Azure SQL database" + ], + "implicit": [ + "The bacpac export failed with SQL71627 partway through, what happened", + "sqlpackage import says the database is not empty and it stopped", + "I want to move this database from the container to a real Azure SQL server", + "Someone asked me to back up the database by exporting a bacpac before a migration" + ], + "negative": [ + "Set up point-in-time restore for my Azure SQL database", + "Bulk load this CSV file into my orders table", + "Build a SQL database project and add a post-deployment script", + "Provision a new Azure SQL Database server and open the firewall" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events deleted file mode 120000 index 9b42126..0000000 --- a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events +++ /dev/null @@ -1 +0,0 @@ -../../../skills/capture-with-extended-events \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/SKILL.md b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/SKILL.md new file mode 100644 index 0000000..3ead4f6 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/SKILL.md @@ -0,0 +1,311 @@ +--- +name: capture-with-extended-events +description: >- + Creates, starts, reads back and drops a database-scoped Extended Events session on Azure SQL + Database, and names the ways one reports success while capturing nothing. Use when someone asks + to capture query text, blocking or deadlocks with Extended Events, XEvents or an XE session on + Azure SQL Database, pastes a session that ran without error and left an empty ring buffer, or + has a session that never fires. Covers ON DATABASE scope and the error ON SERVER raises, the + ring buffer shredded from XML into a rowset, the event_file target's need for a blob URL and a + matching credential, the events and actions the service refuses, and the absence of a built-in + system_health session. Does not diagnose slow queries, blocking or resource pressure once the + data exists (diagnose-slow-query, diagnose-blocking-and-deadlocks, diagnose-resource-pressure) + or read a plan (read-execution-plan). +--- + +# Capture with Extended Events on Azure SQL Database + +Builds a database-scoped event session, starts it, reads its target back as a rowset, and drops it. +What the captured data *means* belongs to the diagnostic skills that consume it. + +Checked 2026-09-03 against Microsoft Learn's Azure SQL Database Extended Events pages and its +Database Engine error tables, and on 2026-08-29 against a local Azure SQL Database container +reporting `EngineEdition` 5. **Where the two disagree this file follows Learn**, because the +container is a different engine build and the subject here is the cloud service. Every +disagreement is named at the point it matters. + +## Four things the service changes, and the error number for each + +| | | +|---|---| +| **Scope** | Sessions are database-scoped only. `ON SERVER` fails with `Msg 25737`, and so does `ON DATABASE` against a system database | +| **Names** | Catalog views and DMVs take a `database_` prefix, never `server_`: `sys.database_event_sessions`, `sys.dm_xe_database_sessions` | +| **Durable target** | `event_file` writes to an Azure Storage blob and nowhere else. A local path fails at `CREATE` with `Msg 40538` | +| **No baseline** | **There is no built-in `system_health` session in Azure SQL Database.** Nothing is capturing anything until you build a session | + +That last row changes behaviour. On SQL Server an agent leans on `system_health` for deadlocks and +skips building anything; here Learn states there is no such session, so there is no fallback and no +`master` connection that reveals one. There *is* an internal `dl` session, and Learn's instruction +is not to read it: `sys.fn_xe_file_target_read_file` over a large `dl` file can raise an +out-of-memory error in `master` and disrupt login processing. + +## Steps + +### 1. Write the session, with the `ACTION()` clauses attribution needs + +An event carries only its own native fields. Who ran the statement, from what application, and the +statement text are *actions*, and an event added without them captures none of the three. + +```sql +CREATE EVENT SESSION capture_statements ON DATABASE +ADD EVENT sqlserver.sql_statement_completed ( + ACTION (sqlserver.sql_text, sqlserver.username, sqlserver.client_app_name) + WHERE duration > 500000 +) +ADD TARGET package0.ring_buffer (SET max_memory = 4096) +WITH (MAX_MEMORY = 4 MB, MAX_DISPATCH_LATENCY = 5 SECONDS, STARTUP_STATE = OFF); +``` + +Two details in that statement earn their place: + +- **`duration > 500000` is half a second, not eight minutes.** `sql_statement_completed` and + `rpc_completed` report `duration` in microseconds; `wait_info` and `wait_completed` report it in + milliseconds. A threshold copied between them is wrong by a factor of 1000 and the session gives + no sign of it. Confirm the unit for the event you are actually using before you write a + predicate, with the query in step 5. +- **`STARTUP_STATE`** decides whether the session comes back after a failover: Learn's guidance is + `ON` for a continuous session, `OFF` for ad hoc troubleshooting. A ring buffer is cleared whenever + the session stops, so `ON` restarts the session and still loses the events. + +### 2. Create and start it on a connection to the user database + +Creating a session does not start it. Run both, on one connection, to the user database and never +to `master`: + +```bash +cat > capture-session.sql <<'SQL' +ALTER EVENT SESSION capture_statements ON DATABASE STATE = START; +SQL + +sqlcmd -S .database.windows.net -d -G -N -b -i capture-session.sql +``` + +`-G` selects Microsoft Entra authentication (`-U -P` instead, for a SQL authentication +login), `-N` requests an encrypted connection, and `-b` makes sqlcmd exit non-zero on error, so a +failed `START` fails the script instead of scrolling past. + +Creating the session needs `CREATE ANY DATABASE EVENT SESSION` in the database and starting it +needs `ALTER ANY DATABASE EVENT SESSION`. These are the database-scoped permissions, not the +`ALTER ANY EVENT SESSION` server permission an example written for SQL Server will ask for. Both +are included in `CONTROL` on the database, held by `dbo`, by `db_owner` and by the server +administrator. + +### 3. Read the ring buffer back as a rowset + +`target_data` arrives as XML. Shred it rather than eyeballing it: + +```bash +sqlcmd -S .database.windows.net -d -G -N -b -Q " +WITH RingBuffer AS ( + SELECT CAST(xst.target_data AS xml) AS TargetData + FROM sys.dm_xe_database_session_targets AS xst + JOIN sys.dm_xe_database_sessions AS xs ON xst.event_session_address = xs.address + WHERE xs.name = N'capture_statements' AND xst.target_name = N'ring_buffer' +), +EventNode AS ( + SELECT CAST(n.NodeData.query('.') AS xml) AS EventInfo + FROM RingBuffer AS rb CROSS APPLY rb.TargetData.nodes('/RingBufferTarget/event') AS n(NodeData) +) +SELECT EventInfo.value('(event/@timestamp)[1]', 'datetimeoffset') AS event_time, + EventInfo.value('(event/data[@name=\"duration\"]/value)[1]', 'bigint') AS duration, + EventInfo.value('(event/action[@name=\"username\"]/value)[1]', 'sysname') AS username, + EventInfo.value('(event/action[@name=\"client_app_name\"]/value)[1]', 'nvarchar(128)') AS app_name, + EventInfo.value('(event/action[@name=\"sql_text\"]/value)[1]', 'nvarchar(max)') AS sql_text +FROM EventNode +ORDER BY event_time DESC;" +``` + +The three `action` columns come back `NULL` for a session built without the `ACTION()` clause in +step 1. Event counts are identical either way, so this is the only place the difference shows. + +### 4. Stop and drop when the capture is done + +```sql +ALTER EVENT SESSION capture_statements ON DATABASE STATE = STOP; +DROP EVENT SESSION capture_statements ON DATABASE; +``` + +Stopping empties a ring buffer, so read it first. A stopped session still appears in +`sys.database_event_sessions` and disappears from `sys.dm_xe_database_sessions`; only `DROP` removes +the definition. Sessions are database-scoped, so dropping the database takes them with it. + +### 5. The three metadata queries worth keeping + +```sql +-- the duration unit for one event, read rather than assumed +SELECT o.name AS event_name, c.name AS column_name, c.description +FROM sys.dm_xe_objects AS o +JOIN sys.dm_xe_object_columns AS c ON c.object_name = o.name AND c.object_package_guid = o.package_guid +WHERE o.name IN (N'sql_statement_completed', N'wait_info') AND c.name = N'duration'; + +-- whether an event, action or target exists on this platform at all +SELECT o.object_type, o.name, o.description +FROM sys.dm_xe_objects AS o +WHERE o.object_type IN ('action', 'event', 'target') AND o.name LIKE '%deadlock%'; + +-- memory this database is spending on started sessions, against the 128 MB cap +SELECT name AS session_name, total_buffer_size + total_target_memory AS total_session_memory +FROM sys.dm_xe_database_sessions; +``` + +Session memory is capped at 128 MB per single database and 512 MB across an elastic pool, and +*started* sessions at 100 per database or pool. Exceeding the memory cap raises `Msg 25746` or +`Msg 25747`, which name `sys.dm_xe_database_sessions` and tell you to stop or shrink a session. + +## Check it worked + +Run this immediately after step 2. It is four assertions with stated expected values, and each one +distinguishes a specific silent failure from success. + +```sql +SET NOCOUNT ON; +DECLARE @name sysname = N'capture_statements', @fail int = 0, @n int; + +-- 1. the definition exists. Zero here means CREATE never ran, or ran in another database +SELECT @n = COUNT(*) FROM sys.database_event_sessions WHERE name = @name; +IF @n <> 1 BEGIN PRINT 'FAIL 1: no session definition. CREATE did not run in this database'; SET @fail = 1; END + +-- 2. it is STARTED. The catalog view above says nothing about this, and this is the usual answer +-- to "the session exists and the ring buffer is empty" +SELECT @n = COUNT(*) FROM sys.dm_xe_database_sessions WHERE name = @name; +IF @n <> 1 BEGIN PRINT 'FAIL 2: defined but never started. Run ALTER EVENT SESSION capture_statements ON DATABASE STATE = START'; SET @fail = 1; END + +-- 3. the actions are attached. Expect 3: sql_text, username, client_app_name +SELECT @n = COUNT(*) +FROM sys.database_event_session_actions AS a +JOIN sys.database_event_sessions AS s ON s.event_session_id = a.event_session_id +WHERE s.name = @name; +IF @n <> 3 BEGIN PRINT CONCAT('FAIL 3: ', @n, ' actions attached, expected 3. Captured rows will have no caller'); SET @fail = 1; END + +-- 4. the target is processing events. Zero AFTER a workload that should match the predicate means +-- the predicate, the unit, or the event choice is wrong, not the session +SELECT @n = ISNULL(SUM(execution_count), 0) FROM sys.dm_xe_database_session_targets AS t +JOIN sys.dm_xe_database_sessions AS s ON s.address = t.event_session_address WHERE s.name = @name; +IF @n = 0 BEGIN PRINT 'WARN 4: target has processed nothing yet. Expected after a matching workload'; END + +IF @fail = 1 THROW 50001, 'event session is not capturing', 1; +PRINT 'event session: DEFINED, STARTED, ACTIONS ATTACHED'; +``` + +Assertions 1 and 2 are the pair that gets read backwards. The catalog view lists every session +*definition* whether or not it runs; the DMV lists only *started* sessions. Read one for the other +and you get the opposite of the truth about whether a capture is live. + +## Sessions that create, start and capture nothing + +Three cases, and none of them raise an error at any point. + +- **No `ACTION()` clause.** Events fire and are counted, and every attribution column is `NULL`. + Assertion 3 above is what catches it. +- **`blocked_process_report`.** Its threshold is set with + `sp_configure 'blocked process threshold'`, an option Learn documents for SQL Server only, and + **which refusal you get depends on which database you are connected to, so quote the one you are + on.** Measured 2026-09-05 on a real logical server: there is no `sp_configure` at all, qualified or + not, and the answer is `Msg 2812, Could not find stored procedure 'sp_configure'`. The container + answers the same way from a user database, re-measured 2026-09-06 on 18.0.226_4_147, which is a + correction: this skill said the container differed. Connected to `master` the procedure is there + and the answer is `Msg 15123, The configuration option 'blocked process threshold' does not exist`, + because that option is not one this engine carries. `Msg 40510, Statement 'CONFIG' is not supported + in this version of SQL Server` comes from `RECONFIGURE` and not from `sp_configure`, and an option + the engine does carry stages happily before `RECONFIGURE` refuses to apply it. Either way + `sys.database_scoped_configurations` has no equivalent key. Measured on the container: this event + created, started and captured zero across a 28 second real lock wait. **The supported route to + blocked process reports here is not an event session at all**: enable the `Blocks` diagnostic + settings category and read `blocked_process_filtered_s` out of the streamed resource logs. +- **An event or action the platform does not carry.** These are rejected at `CREATE` rather than + silently: `Msg 25742` for a target, `Msg 25743` for an event, `Msg 25744` for an action. Check + with the second query in step 5 first. + +**On deadlocks, Learn and the container disagree, and this file follows Learn.** Learn's documented +way to capture deadlock graphs on Azure SQL Database is a database-scoped session on +`sqlserver.database_xml_deadlock_report`. On the container that event created, started and captured +zero events across two confirmed deadlocks (`Msg 1205` each time), while `sqlserver.lock_deadlock` +captured one. That gap has not been reproduced on the cloud service and is not evidence about it, +so build the session Learn documents and verify it with assertion 4 above against a deadlock you +cause deliberately. Keep `lock_deadlock` as the fallback only if that verification comes back empty. + +When a session runs clean and its target stays empty, open +[references/sessions-that-capture-nothing.md](references/sessions-that-capture-nothing.md) for the +captured XML of each case above, the container measurements behind them, and the +`sys.dm_xe_object_columns` duration text for a specific event. + +## The `event_file` target, in one paragraph + +A local filesystem path is rejected at `CREATE`, not left unwritten: +`Msg 40538, A valid URL beginning with 'https://' is required as value for any filepath specified`. +That same message is also what a **malformed blob URL** gets, and it names the wrong thing: measured +on a logical server 2026-09-08, a container segment of fewer than three characters is refused with +`Msg 40538` while the identical URL with a longer container name is accepted, so a reader retypes a +scheme that was already `https://`. Check the container name against Azure's blob naming rules +before touching the scheme. + +An `https://` blob URL is accepted at `CREATE` with no credential check at all; the credential is +evaluated only at `START`. **Expect `Msg 25602` there, not `Msg 25739`.** `Msg 25739` exists and its +text is exactly this condition, "required credential for writing session output to Azure blob is +missing", and the engine does not raise it: measured 2026-09-08 on a logical server, four times, +against both a fictitious storage account and a real container with no credential, every start +answered `Msg 25602` with operating system error 86, "The specified network password is not +correct". That error text sends you after a password when what is missing is the credential object. +The most common cause of `Msg 25602` is documented and specific: +**the database scoped credential's name must be the blob container URL itself, with no trailing +slash.** Open +[references/event-file-to-blob-storage.md](references/event-file-to-blob-storage.md) before you +choose `event_file` over the ring buffer, because it carries the credential and managed identity +syntax and the table that maps each operating system error number at `START` to its cause. + +## Validation rules + +- The session is created `ON DATABASE`, on a connection to the user database, never `ON SERVER` and + never against `master`. +- Every event that will be attributed or filtered by caller carries the `ACTION()` clauses that + attribution needs, and assertion 3 in "Check it worked" confirms the count. +- The session is confirmed present in `sys.dm_xe_database_sessions`, not merely in + `sys.database_event_sessions`, before an empty target is believed. +- Any `duration` predicate names the event it was written for, and its unit came from + `sys.dm_xe_object_columns`. +- `STARTUP_STATE` is a deliberate choice, and a ring buffer is read before the session is stopped. +- An `event_file` target points at an `https://` blob URL, and access was proved by a successful + `START`, never by the `CREATE` succeeding. +- Started sessions and their memory were checked against the 100 session and 128 MB per database + limits before another was added, and every session built by hand is dropped when it is done. + +## Do not + +- Do not look for a `system_health` session on Azure SQL Database. There is not one, and the time + spent hunting is time nothing is being captured. +- Do not read the internal `dl` event session. A large read from it can raise an out-of-memory + error in `master` and affect login processing. +- Do not scope a session `ON SERVER` for Azure SQL Database. The fix is `ON DATABASE`, not a + permissions escalation. +- Do not grant `ALTER ANY EVENT SESSION` and expect it to work. The database-scoped permissions are + the `ANY DATABASE EVENT SESSION` family. +- Do not reuse a `duration` threshold across events. Microseconds and milliseconds both appear, and + the wrong one fails silently in whichever direction is less noticeable. +- Do not point `event_file` at a local path. It is refused at `CREATE` and no retry changes that. +- Do not treat a successful `CREATE` with a blob target as evidence the credential works, and do not + name that credential after the session or the storage account. Its name must be the container URL, + and a trailing slash on it is its own failure. +- Do not build a blocking diagnosis on `blocked_process_report` on Azure SQL Database. Use the + `Blocks` diagnostic settings category, which is the supported source for that report. +- Do not restate what the captured queries, waits or blocking chains mean. That belongs to + diagnose-slow-query, diagnose-blocking-and-deadlocks and diagnose-resource-pressure. + +## References + +- [references/sessions-that-capture-nothing.md](references/sessions-that-capture-nothing.md) when a + session runs clean and its target stays empty: the container measurements behind each case, the + captured XML, the duration column text for the events named here, and the `ACTION()` comparison + run side by side. +- [references/event-file-to-blob-storage.md](references/event-file-to-blob-storage.md) before + choosing `event_file` over the ring buffer, and again if `START` fails: the credential and managed + identity syntax, the SAS token permissions, the operating system error numbers at `START` and what + each one means, and reading `.xel` files back. +- [Extended Events in Azure SQL](https://learn.microsoft.com/azure/azure-sql/database/xevent-db-diff-from-svr): + scope, permissions, storage authorization and the resource limits quoted above. Read it before + trusting any Extended Events example not written for this platform. +- [Collect deadlock graphs in Azure SQL Database](https://learn.microsoft.com/azure/azure-sql/database/analyze-prevent-deadlocks): + the documented deadlock session, both targets, and the query that shreds a graph out of a ring + buffer. Read it when the capture you need is deadlocks. +- `diagnose-slow-query`, `diagnose-blocking-and-deadlocks`, `diagnose-resource-pressure`: what the + captured data means, once this skill has produced it. +- `read-execution-plan`: the plan for a statement this session identified. diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/event-file-to-blob-storage.md b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/event-file-to-blob-storage.md new file mode 100644 index 0000000..521ab35 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/event-file-to-blob-storage.md @@ -0,0 +1,111 @@ +# The event_file target: reaching blob storage, and reading it back + +From Microsoft Learn's Azure SQL Database Extended Events pages and its Database Engine error +tables, checked 2026-09-03, except where a line says it was measured. The container this repository +tests against has no Azure Storage account, so **no write to a real blob and no read back from one +was exercised here.** Treat the syntax as documented, not measured, and prove it with one successful +`START` against your own storage account. + +## Contents + +- [Storage account settings that fail at START](#storage-account-settings-that-fail-at-start) +- [The credential name is the container URL](#the-credential-name-is-the-container-url) +- [Managed identity, the option with no expiry](#managed-identity-the-option-with-no-expiry) +- [SAS token, the option with a secret](#sas-token-the-option-with-a-secret) +- [Every error START can raise](#every-error-start-can-raise-and-what-each-one-means) +- [Reading the .xel files back](#reading-the-xel-files-back) + +`event_file` is the only target that survives a session stop or a failover, and "durable" here means +an Azure Storage blob specifically. There is no local disk to fall back to: a local path is refused +at `CREATE` with `Msg 40538`. + +## Storage account settings that fail at START + +**Hierarchical namespace enabled**, and **an immutability policy on the container**, are both +unsupported and both surface as operating system error 13. **The storage account firewall must be +off** if you authenticate with a SAS token, which Learn states plainly is unsupported behind a +firewall. The full account requirements, including tier, region and redundancy matching, are on +[Create an event session with an event_file target in Azure Storage](https://learn.microsoft.com/azure/azure-sql/database/xevent-code-event-file); +read them there rather than from a copy that can go stale. + +## The credential name is the container URL + +**The database scoped credential must be named exactly the blob container URL, with no trailing +slash.** Not the session name, not the storage account name, not a friendly label. A name that does +not match the container URL is the documented first cause of operating system error 86 at `START`, +and a trailing slash on an otherwise correct name is the documented second. + +```sql +-- the name is the container URL. The event_file filename then points INSIDE that container. +CREATE DATABASE SCOPED CREDENTIAL + [https://.blob.core.windows.net/] +WITH IDENTITY = 'MANAGED IDENTITY'; +``` + +Creating the credential requires `CONTROL` on the database, held by `dbo`, `db_owner` and the +server administrator. Creating the *event session* afterwards does not, so an administrator can +create the credential once and a lower-privileged principal can create sessions against it later. + +## Managed identity, the option with no expiry + +Assign the **Storage Blob Data Contributor** role on the container to the managed identity of the +logical server: the system assigned identity if it is enabled, otherwise the user assigned identity +designated as primary. Create the credential with `WITH IDENTITY = 'MANAGED IDENTITY'` and no +secret, as above. Prefer this path for anything meant to keep running, because there is no SAS token +to expire underneath it. + +## SAS token, the option with a secret + +Three requirements, all exact, or `START` fails: permissions `rwdl` (`Read`, `Write`, `Delete`, +`List`, and Read plus Write alone is not enough); start and expiry times spanning the whole lifetime +of the event session; and **no IP address restrictions**. + +```sql +IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##') + CREATE MASTER KEY; + +CREATE DATABASE SCOPED CREDENTIAL + [https://.blob.core.windows.net/] +WITH IDENTITY = 'SHARED ACCESS SIGNATURE', + SECRET = ''; +``` + +The master key comes first: without one, the credential create fails with `Msg 15581`, asking for a +master key in the database, measured on the container. Treat the SAS token as a password. + +## Every error START can raise, and what each one means + +`CREATE EVENT SESSION` with an `https://` blob target succeeds with no credential check at all. The +credential is not evaluated until `START`, which is where all of this surfaces: + +| At `START` | Meaning | +|---|---| +| `Msg 25739` | No credential at all for writing session output to the blob | +| `Msg 40539` | The named Azure Storage credential was not found | +| `Msg 25602` + operating system error 86, "The specified network password is not correct" | No credential whose **name matches the container URL**, or a credential name ending in a slash | +| `Msg 25602` + operating system error 5, "Access is denied" | Managed identity missing the role assignment; or the storage firewall is on and either the `Microsoft.Sql/servers` resource instance is not granted access, or SAS token authentication is in use, which the firewall does not support | +| `Msg 25602` + operating system error 3, "The system cannot find the path specified" | The container in the URL does not exist | +| `Msg 25602` + operating system error 13, "The data is invalid" | An immutability policy on the container, or hierarchical namespace enabled on the account | +| `Msg 25738` or `Msg 25740` | The system is busy. Retry later; nothing is misconfigured | + +Operating system error 86 names a Windows network error because the engine is reporting the file +system layer under the blob mount. There is no password to fix. + +## Reading the .xel files back + +```sql +SELECT CAST(event_data AS xml) AS event_xml, file_name, file_offset, timestamp_utc +FROM sys.fn_xe_file_target_read_file( + 'https://.blob.core.windows.net//capture_statements*.xel', + NULL, NULL, NULL); +``` + +The wildcard matters: `event_file` rolls to a new blob once `max_file_size` is reached, and one +call reads every rollover file. The three trailing `NULL` arguments are a metadata file, a file to +resume from, and an offset; leave all three `NULL` for a full read. Reading from a database other +than the one that wrote the files needs that database to hold its own credential on the container. + +**A wrong or missing filename returns zero rows, with no error of any kind**, measured on the +container. So does a read blocked by a network security perimeter, which may instead raise +`Msg 25759` or `Msg 25717`. Before concluding a capture was empty, confirm the blob itself exists +through the storage account rather than trusting a zero-row result from this function. diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/sessions-that-capture-nothing.md b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/sessions-that-capture-nothing.md new file mode 100644 index 0000000..e8de24e --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/references/sessions-that-capture-nothing.md @@ -0,0 +1,174 @@ +# Event sessions that create, start, and capture nothing + +The evidence behind the empty-capture cases in SKILL.md. + +## Contents + +- [Duration units, event by event](#duration-units-event-by-event) +- [ACTION versus no ACTION](#action-versus-no-action-same-event-side-by-side) +- [blocked_process_report captured zero](#blocked_process_report-captured-zero) +- [database_xml_deadlock_report captured zero, and Learn disagrees](#database_xml_deadlock_report-captured-zero-and-learn-disagrees) +- [lock_deadlock captured one](#lock_deadlock-captured-one) +- [There is no system_health session](#there-is-no-system_health-session-to-fall-back-on) + +**Read the provenance before you use a number here.** All of it was measured on 2026-08-29 against +a local Azure SQL Database container reporting `EngineEdition` 5, in a scratch database. That is a +different engine build from the cloud service, so where a measurement here contradicts Microsoft +Learn, **the contradiction is stated and Learn wins**. + +## Duration units, event by event + +Read from `sys.dm_xe_object_columns`, which carries a `description` for every column of every event: + +| Event | `duration` column description | +|---|---| +| `sql_statement_completed` | "The time (in microseconds) that it took to execute the statement." | +| `rpc_completed` | "The time (in microseconds) that the remote procedure call took to be completed." | +| `lock_acquired` | "The time (in microseconds) between when the lock was requested and when it was canceled." | +| `wait_completed` | "Wait duration in milliseconds" | +| `wait_info` | "Wait duration in milliseconds" | + +Confirmed empirically for `sql_statement_completed`: a session filtering `WHERE duration > 500000` +on that event, with `WAITFOR DELAY '00:00:01'` run against it, captured the statement with +`duration` = `1000292`. A microsecond reading for a one-second wait fits; a millisecond reading +would have been roughly `1000`. + +The failure this produces has no symptom. A predicate copied from a `wait_info` session onto a +`sql_statement_completed` session is off by a factor of 1000: it captures far more than intended, +or, more often, nothing at all. + +## ACTION versus no ACTION, same event side by side + +Two sessions, both on `sqlserver.sql_statement_completed`, both targeting `package0.ring_buffer`, +run against the same kind of statement. + +With actions, `ACTION (sqlserver.sql_text, sqlserver.username, sqlserver.client_app_name)`, the +captured XML carried, alongside the native fields: + +```xml +SQLCMD +sa +SET TEXTSIZE 4096 +``` + +Without the `ACTION()` clause the same event shape came back, `statement` field and all, with +**zero** `` elements: no error, no warning, and an identical `eventCount`. Nothing +distinguishes them but the presence of an `` element, which is why "Check it worked" counts +rows in `sys.database_event_session_actions` instead. + +## blocked_process_report captured zero + +Setup: a table with one row; one session holding an open transaction with an `UPDATE` on that row +for roughly 30 seconds; a second session issuing a conflicting `UPDATE` on the same row shortly +after. Both completed without error, confirming the second genuinely waited behind the first's lock +for most of that window. + +```sql +CREATE EVENT SESSION dq_xe_blocking ON DATABASE +ADD EVENT sqlserver.blocked_process_report +ADD TARGET package0.ring_buffer; +ALTER EVENT SESSION dq_xe_blocking ON DATABASE STATE = START; +``` + +Read back after the blocking episode finished: + +```xml + +``` + +`execution_count` for the target was also `0`, and the session was present in +`sys.dm_xe_database_sessions` throughout. The usual fix, setting the threshold, is not available: + +On a **real logical server**, measured 2026-09-05 on `sqllab200068e2`, Basic tier, there is no +`sp_configure` to call at all: + +```output +EXEC sys.sp_configure N'blocked process threshold', 5; +Msg 2812, Level 16, State 62 +Could not find stored procedure 'sys.sp_configure'. +``` + +`OBJECT_ID('sys.sp_configure')` is `NULL` there, and the unqualified name answers the same way. + +On the **container** the procedure exists and refuses the statement instead: + +```output +EXEC sp_configure 'blocked process threshold'; +Msg 40510, Level 16, State 1 +Statement 'CONFIG' is not supported in this version of SQL Server. +``` + +Both are refusals and they are not the same refusal, so an agent matching on `40510` will not +recognise the cloud one. `sys.database_scoped_configurations` has no row matching `%BLOCK%` or +`%THRESHOLD%` on either. Learn corroborates the option itself: `blocked process threshold` is +documented for SQL Server only, and 40510 is in the error table with exactly that message. The event is not hard to trigger here, there +is no exposed control to trigger it with. + +Learn also names the supported route on Azure SQL Database, which is not an event session: the +`Blocks` resource log category, streamed through a diagnostic setting, carries +`blocked_process_filtered_s` (the blocked process report XML), `lock_mode_s`, +`resource_owner_type_s` and `duration_d` in microseconds. + +## database_xml_deadlock_report captured zero, and Learn disagrees + +Setup: two tables, each with one row. Two sessions each update their own table, wait three seconds, +then attempt to update the other's, guaranteeing a lock-order deadlock. Run twice, independently. + +```sql +CREATE EVENT SESSION dq_xe_deadlock ON DATABASE +ADD EVENT sqlserver.database_xml_deadlock_report +ADD TARGET package0.ring_buffer; +ALTER EVENT SESSION dq_xe_deadlock ON DATABASE STATE = START; +``` + +Both runs produced a real deadlock, each confirmed by the victim receiving `Msg 1205, Level 13`, +naming it the deadlock victim. Both times the ring buffer read back with `eventCount="0"`. `sys.dm_xe_objects` describes the event +as "Produces a deadlock report for a victim, with information scoped to the victim's database" with +`capabilities_desc` = `sds_visible`, so the engine reported it as valid and visible in this scope. +It was created, attached and started without error at any step, and simply did not fire. + +**Microsoft Learn documents this exact event as the way to collect deadlock graphs on Azure SQL +Database**, in both the ring buffer and event file forms, with `STARTUP_STATE = ON` and +`MAX_MEMORY = 4 MB`. That is a direct contradiction of the measurement above, and it is not +resolved: the measurement is from the container, not from the cloud service, and nothing here +tests the cloud service. Follow Learn, build the session it documents, and verify it against a +deadlock you cause on purpose before relying on it. Learn also notes that optimized locking is +always on in Azure SQL Database, which makes deadlocks less likely in the first place, so an empty +session is not by itself evidence the event is broken. + +## lock_deadlock captured one + +Same deadlock setup, a different database-scoped event: + +```sql +CREATE EVENT SESSION dq_xe_deadlock2 ON DATABASE +ADD EVENT sqlserver.lock_deadlock +ADD TARGET package0.ring_buffer; +ALTER EVENT SESSION dq_xe_deadlock2 ON DATABASE STATE = START; +``` + +captured one event on the first attempt, carrying `resource_type` `XACT`, `mode` `S`, `owner_type` +`Transaction`, a `transaction_id`, a `database_id` and a `deadlock_id`. It gives no deadlock graph, +only the resource, the lock mode, the owning transaction and an id grouping the participants of one +deadlock: enough to confirm a deadlock happened and roughly on what. Keep it as the fallback for +when the documented session verifies empty, not as the first choice. + +## There is no system_health session to fall back on + +An earlier version of this file spent most of its length on `system_health`: the container has one, +visible from `master` through `sys.dm_xe_sessions`, whose ring buffer did capture +`xml_deadlock_report` for a real deadlock. That was correctly measured, and **it is a property of +the container, not of Azure SQL Database.** Learn is unambiguous, and the `system_health` +documentation page applies to SQL Server only: "There's no built-in `system_health` Extended Event +session in Azure SQL Database." There is no baseline session on the cloud service, no `master` +connection that reveals one, and no reason to look. Build the session you need. + +The one built-in session, `dl`, comes with a prohibition rather than an invitation: do not read +deadlock events from it, because `sys.fn_xe_file_target_read_file` over a large `dl` file can raise +an out-of-memory error in `master`, affecting login processing and causing an application outage. + +Also measured on the container: `sys.dm_xe_database_objects` does not exist, raising `Msg 208` +either way. Learn corroborates it, listing the Extended Events DMVs common to Azure SQL Database as +`sys.dm_xe_map_values`, `sys.dm_xe_object_columns`, `sys.dm_xe_objects` and `sys.dm_xe_packages`, +with no database-prefixed variant. Event, action and target metadata comes from plain +`sys.dm_xe_objects`. diff --git a/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/skill.spec.jsonc new file mode 100644 index 0000000..54448ad --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/capture-with-extended-events/skill.spec.jsonc @@ -0,0 +1,166 @@ +// skills/capture-with-extended-events/skill.spec.jsonc +{ + "id": "capture-with-extended-events", + "domain": "performance-and-diagnostics", + "value": ["not-in-training-data", "multi-step-with-silent-failures"], + "correction": "Agents port an Extended Events script from SQL Server onto Azure SQL Database almost unchanged: scoping the session ON SERVER, pointing an event_file target at a local path, omitting ACTION() clauses, and reaching for a system_health session this platform does not have. Several of the resulting failures are not errors. The session is created, starts, and reports itself healthy while capturing nothing useful, or nothing at all.", + "value_probe": { + "gate": "The answer produces a session that is scoped to the user database and carries the clauses that attribution needs, and it sends the reader neither to a built-in baseline session nor to a file path on the machine running the engine.", + "task": "We get a deadlock in our Azure SQL Database maybe twice a day and never while anyone is watching. I want to capture them over the next few days, along with the statement text and which application and which user was running it, so that I have something to look at on Monday morning. I have a script from our old on-premises server that used to do exactly this. Tell me what to actually run against this database, and how to read the results back afterwards.", + "states": [ + "The catalog views and dynamic management views for these sessions carry a database prefix rather than a server one.", + "The attribution fields have to be requested explicitly, because an event carries only its own native fields.", + "A durable file target has to write to Azure Storage, with a credential named for the container.", + "A session can be created, started, and report itself healthy while capturing nothing useful." + ], + "refutes": [ + "A built-in health session on this database already holds the recent deadlocks and can simply be read for them.", + "The session can be created at server scope the way the on-premises script does.", + "The durable file target can be pointed at a local path on the machine running the engine.", + "Adding the event on its own captures the statement text, the user name and the application name." + ] + }, + "posture": ["inspect", "read"], + "maturity": "ga", + "applies_to": ["azure-sql-db"], + "related": ["diagnose-slow-query", "diagnose-blocking-and-deadlocks", "diagnose-resource-pressure", "read-execution-plan"], + "validation": { + "target": "cloud", + "setup": [ + "a dedicated database the agent can create event sessions and tables in", + "a connection holding CONTROL on that database, or CREATE ANY DATABASE EVENT SESSION, ALTER ANY DATABASE EVENT SESSION and VIEW DATABASE PERFORMANCE STATE" + ], + "assert": [ + "CREATE EVENT SESSION probe_on_server ON SERVER ADD EVENT sqlserver.sql_statement_completed ADD TARGET package0.ring_buffer fails with error 25737 on Azure SQL Database", + "sys.dm_xe_database_sessions lists a session only after ALTER EVENT SESSION probe_db_scope ON DATABASE STATE = START, so the catalog view and the DMV answer different questions", + "an event added without an ACTION() clause attaches no rows to sys.database_event_session_actions and captures no username, client_app_name or sql_text", + "an event_file target pointed at a local filesystem path is rejected at CREATE time with error 40538", + "an event_file target pointed at an https blob URL is accepted at CREATE and fails only at START, and the number there is 25602 with operating system error 86, not the 25739 whose own text names this exact condition. Measured 2026-09-08 on sqllab0908614 against both a fictitious storage account and a real container with no credential", + "error 40538 is a URL SHAPE check and its text names only the scheme: a blob container segment of fewer than three characters is refused with 40538 while the same URL with a longer container name is accepted, measured 2026-09-08", + "a database scoped credential is usable by an event_file target only when its name is exactly the blob container URL with no trailing slash", + "the duration field is documented in microseconds for sql_statement_completed and rpc_completed and in milliseconds for wait_info and wait_completed", + "sp_configure 'blocked process threshold' cannot be set on Azure SQL Database, so a blocked_process_report session has no threshold to cross and the supported source for that report is the Blocks resource log category. The refusal differs by engine and the right one has to be quoted: a real logical server has no sp_configure at all and answers Msg 2812, while the container carries the procedure and answers Msg 40510", + "the database-scoped session and memory limits are 100 started sessions and 128 MB per single database, and exceeding the memory limit raises error 25746 or 25747", + "Azure SQL Database has no built-in system_health event session, so there is no baseline capture to fall back on and no master connection that reveals one" + ], + "probes": [ + { + "sql": "CREATE EVENT SESSION probe_on_server ON SERVER\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.ring_buffer;", + "expect": { + "error": 25737 + }, + "why": "Nearly every Extended Events example in the wild is written for SQL Server and scopes the session ON SERVER, so this is the first thing a ported script does. Learn's error table gives 25737, severity 16, as 'Database scoped extended event sessions are not available in server scope or system databases in Azure DB', and the container measured the same on 2026-08-29. This probe is the assert entry that once carried an ellipsis where the statement body belongs, written out." + }, + { + "sql": "CREATE EVENT SESSION probe_event_file_local ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.event_file (SET filename = N'/tmp/probe-capture.xel');", + "expect": { + "error": 40538 + }, + "why": "An event_file target pointed at a local path is rejected at CREATE, not merely left unwritten. Learn's error table gives 40538 as 'A valid URL beginning with https:// is required as value for any filepath specified' and the container measured that number on 2026-08-29. Probing at CREATE is the point, because the blob URL form is accepted at CREATE and fails only at START. Learn lists 25741 for the same condition, so a 25741 here means the product moved rather than the probe being wrong." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_db_scope')\n DROP EVENT SESSION probe_db_scope ON DATABASE;\nCREATE EVENT SESSION probe_db_scope ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.ring_buffer (SET max_memory = 4096);\nSELECT\n (SELECT COUNT(*) FROM sys.database_event_sessions WHERE name = 'probe_db_scope') AS defined,\n (SELECT COUNT(*) FROM sys.dm_xe_database_sessions WHERE name = 'probe_db_scope') AS started;", + "expect": { + "rows": [ + [ + 1, + 0 + ] + ] + }, + "why": "One probe for the two claims that get read backwards. Learn states it directly: catalog views describe session definitions and return nothing about active sessions, while the DMVs return only started sessions. An agent reading the wrong one concludes the opposite of the truth about whether its capture is live. Measured on the container 2026-08-29." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_actions')\n DROP EVENT SESSION probe_actions ON DATABASE;\nCREATE EVENT SESSION probe_actions ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed (\n ACTION (sqlserver.sql_text, sqlserver.username, sqlserver.client_app_name)\n )\n ADD TARGET package0.ring_buffer (SET max_memory = 4096);\nSELECT COUNT(*) AS attached_actions\nFROM sys.database_event_session_actions AS a\nJOIN sys.database_event_sessions AS s ON s.event_session_id = a.event_session_id\nWHERE s.name = 'probe_actions';", + "expect": { + "rows": [ + [ + 3 + ] + ] + }, + "why": "The ACTION() claim made executable. A session with no ACTION() clause fires, counts events and reports itself healthy while carrying no caller, application or statement text, and the event count is identical either way. Counting attached actions in the catalog view is the only cheap signal. Learn documents sys.database_event_session_actions as one row per action per event, joined to sys.database_event_sessions on event_session_id." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_deadlock_capture')\n DROP EVENT SESSION probe_deadlock_capture ON DATABASE;\nCREATE EVENT SESSION probe_deadlock_capture ON DATABASE\n ADD EVENT sqlserver.database_xml_deadlock_report\n ADD TARGET package0.ring_buffer\n WITH (STARTUP_STATE = ON, MAX_MEMORY = 4 MB);\nALTER EVENT SESSION probe_deadlock_capture ON DATABASE STATE = START;\nSELECT COUNT(*) AS started FROM sys.dm_xe_database_sessions WHERE name = 'probe_deadlock_capture';", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The one disagreement in this skill that only a live cloud run can settle. Learn documents exactly this session as the way to collect deadlock graphs on Azure SQL Database, while the container measured the event creating, starting and capturing zero across two confirmed deadlocks on 2026-08-29. This asserts the half that is settleable without causing a deadlock: the event is accepted rather than refused with error 25743, and the session reaches a started state. A failure here means Learn's own documented recipe does not run." + }, + { + "sql": "EXEC sys.sp_configure N'blocked process threshold', 5;", + "expect": { + "error": 2812 + }, + "why": "Why a blocked_process_report session creates, starts, reports healthy and captures nothing across a real 28 second lock wait: there is no threshold to cross, not a threshold set too high. This probe expects Msg 2812, Could not find stored procedure, because validation.target here is cloud and that is what a real logical server answers, measured 2026-09-05 on sqllab200068e2: OBJECT_ID('sys.sp_configure') is NULL and both the qualified and unqualified names raise 2812. It expected 40510 until that run. 40510 is not wrong, it is the CONTAINER answer, measured 2026-08-29 naming CONFIG, and the two engines refuse this differently because one carries the procedure and the other does not. Red if a logical server ever grows an sp_configure, which would change the advice. Without this probe the empty ring buffer reads as a tuning problem." + } + , + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_bad_target')\n DROP EVENT SESSION probe_bad_target ON DATABASE;\nCREATE EVENT SESSION probe_bad_target ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.etw_classic_sync_target;", + "expect": { + "error": 25742 + }, + "why": "The first of the three refusals a ported SQL Server script meets, and the skill states all three in one sentence with none of them run. Measured 2026-09-08 on sqllab0908614: Target 'package0.etw_classic_sync_target' is not available for Azure SQL Database. Red if the service ever admits a Windows-only target, which would change what the skill says about porting. It catches on the number, so a typo in the target name reports its own number and goes red rather than passing." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_bad_event')\n DROP EVENT SESSION probe_bad_event ON DATABASE;\nCREATE EVENT SESSION probe_bad_event ON DATABASE\n ADD EVENT sqlserver.xml_deadlock_report\n ADD TARGET package0.ring_buffer;", + "expect": { + "error": 25743 + }, + "why": "The server-scoped deadlock event every on-premises script uses, and the reason this skill sends a reader to database_xml_deadlock_report instead. Measured 2026-09-08 on sqllab0908614: The event 'sqlserver.xml_deadlock_report' is not available for Azure SQL Database. Red if it becomes available, which would remove the reason for the substitution the skill teaches." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_bad_action')\n DROP EVENT SESSION probe_bad_action ON DATABASE;\nCREATE EVENT SESSION probe_bad_action ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed (ACTION (sqlserver.server_instance_name))\n ADD TARGET package0.ring_buffer;", + "expect": { + "error": 25744 + }, + "why": "The third refusal, and the one that lands inside the ACTION() clause this skill spends a step telling the reader to add. Measured 2026-09-08 on sqllab0908614: The action 'sqlserver.server_instance_name' is not available for Azure SQL Database. Red if a server-scoped action becomes addable at database scope." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_memory_cap')\n DROP EVENT SESSION probe_memory_cap ON DATABASE;\nCREATE EVENT SESSION probe_memory_cap ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.ring_buffer\n WITH (MAX_MEMORY = 1048576 KB);", + "expect": { + "error": 25746 + }, + "why": "The 128 MB per database event session memory ceiling, which the skill states as a number and nothing checked. One gibibyte asked for in a single session is comfortably past it on any tier, so this does not depend on what else is running. Measured 2026-09-08 on sqllab0908614: Operation will cause database event session memory to exceed allowed limit. Red if the ceiling moves, which would make the number in the body wrong." + }, + { + "sql": "IF EXISTS (SELECT 1 FROM sys.database_event_sessions WHERE name = 'probe_blob_nocred')\n DROP EVENT SESSION probe_blob_nocred ON DATABASE;\nCREATE EVENT SESSION probe_blob_nocred ON DATABASE\n ADD EVENT sqlserver.sql_statement_completed\n ADD TARGET package0.event_file (SET filename = 'https://nosuchaccount0908.blob.core.windows.net/xeprobes/probe');\nALTER EVENT SESSION probe_blob_nocred ON DATABASE STATE = START;", + "expect": { + "error": 25602 + }, + "why": "The claim this skill had backwards until 2026-09-08. A well formed blob URL is accepted at CREATE with no credential check, and the START then answers Msg 25602 with operating system error 86, not the Msg 25739 whose own sys.messages text names this exact condition. Measured four times that day on sqllab0908614, twice against this fictitious account and twice against a real storage container created for the run, all four identical. The filename deliberately carries no .xel extension and a container segment of three characters or more, because the engine validates the URL against blob naming rules at CREATE and reports every shape failure as Msg 40538. Red if the engine starts raising 25739 here, which would mean the product moved and the skill's original wording was right after all." + }, + { + "where": "host", + "cmd": "set -e; Q() { sqlcmd -S \"$PROBE_SERVER\" -d \"$PROBE_DB\" --authentication-method=ActiveDirectoryAzCli -b -I -m-1 -W -l 30 -Q \"$1\" 2>&1; }; Q \"DROP TABLE IF EXISTS dbo.probe_dl1; DROP TABLE IF EXISTS dbo.probe_dl2; CREATE TABLE dbo.probe_dl1(i INT); CREATE TABLE dbo.probe_dl2(i INT); INSERT dbo.probe_dl1 VALUES(1); INSERT dbo.probe_dl2 VALUES(1);\" > /dev/null; o=$(mktemp -d); Q \"BEGIN TRAN; UPDATE dbo.probe_dl1 SET i=i+1; WAITFOR DELAY '00:00:03'; UPDATE dbo.probe_dl2 SET i=i+1; COMMIT;\" > \"$o/a\" 2>&1 & sleep 1; Q \"BEGIN TRAN; UPDATE dbo.probe_dl2 SET i=i+1; WAITFOR DELAY '00:00:03'; UPDATE dbo.probe_dl1 SET i=i+1; COMMIT;\" > \"$o/b\" 2>&1 & wait; grep -q 'Msg 1205,' \"$o/a\" \"$o/b\"", + "expect": { + "succeeds": true + }, + "why": "Msg 1205 is the number this skill's whole deadlock section rests on and no sql probe can produce it, because a deadlock needs two sessions and the lane hands a probe one batch. So it is a host probe that opens two Entra connections of its own and takes two tables in opposite order. The grep is for the literal string Msg 1205, with its comma, so a different error in either session leaves the grep with no match and the probe red: this is not a TRY/CATCH that passes on any failure. Measured 2026-09-08 on sqllab0908614: Transaction (Process ID 75) was deadlocked on lock resources with another process and has been chosen as the deadlock victim, Level 13, State 72. Red if the deadlock monitor stops choosing a victim, or if 1205 changes." + } + ] + }, + "triggering": { + "explicit": [ + "Set up an Extended Events session to capture slow queries on my Azure SQL database", + "How do I create an XE session on Azure SQL Database", + "Capture blocking with Extended Events on Azure SQL" + ], + "implicit": [ + "I created an event session and started it but the ring buffer is empty", + "My deadlock event session never fires even though I know the app is deadlocking", + "Where do I find system_health on my Azure SQL database", + "I got Invalid object name sys.dm_xe_sessions when I tried to check system_health, so I guess Azure SQL Database does not have it" + ], + "negative": [ + "Read the estimated execution plan for this query", + "Why is my query blocked right now, what is holding the lock", + "Turn on Query Store for this database" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks deleted file mode 120000 index c77ea88..0000000 --- a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks +++ /dev/null @@ -1 +0,0 @@ -../../../skills/diagnose-blocking-and-deadlocks \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/SKILL.md b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/SKILL.md new file mode 100644 index 0000000..49a8560 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/SKILL.md @@ -0,0 +1,210 @@ +--- +name: diagnose-blocking-and-deadlocks +description: >- + Finds who is blocking whom on Azure SQL Database right now, and reads a completed deadlock + graph out of the database-scoped Extended Events session that captured it. Use when someone + reports a query or app that hangs under load, pastes "Msg 1205" or "was deadlocked on lock + resources", asks who is blocking a session, or ran a blocking query that came back empty and + assumed nothing was blocked. Covers which grant a login needs to see another session's blocking + and why that differs on Basic, S0, S1 and elastic pools, why an idle session holding a + transaction is a head blocker that never appears in sys.dm_exec_requests, and how optimized + locking's wait types differ. Does not tune an unblocked query (diagnose-slow-query), diagnose + CPU, memory or IO pressure (diagnose-resource-pressure), build or repair the session + (capture-with-extended-events) or read the plan (read-execution-plan). +--- + +# Diagnose blocking and read a deadlock on Azure SQL Database + +**Two empty results, two different causes, and the fix for one does nothing for the other.** The +blocking DMVs answer only as far as the login's grant reaches. Extended Events does not work the way +SQL Server taught you: Microsoft Learn states there is +[no built-in `system_health` session in Azure SQL Database](https://learn.microsoft.com/sql/relational-databases/extended-events/use-the-system-health-session#the-system_health-session-in-azure-sql) +and that sessions are database-scoped, so nothing captured your deadlock unless someone built a +session first. + +| What you see | Cause | Fix | +|---|---|---| +| one row, `blocking_session_id = 0`, no error, while a real chain runs | the login cannot see past its own session | section 1, and which grant depends on the service objective | +| `Msg 208, Invalid object name 'sys.dm_xe_sessions'` | there are no server-scoped Extended Events views here | section 5. No grant and no `master` produces them | + +No permission moves the second one: +[`VIEW SERVER STATE` cannot be granted in Azure SQL Database](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-requests-transact-sql#permissions) +at all. + +Sections 1 to 4 and 6 were measured 2026-08-29 on **the local Azure SQL Database container** +(`EngineEdition` 5, `Edition` `SQL Azure`) and each is cited to Learn below. Section 5 is not: the +container carries a `master` and a `system_health` session the cloud service does not, and reading +deadlocks out of it is the mistake this page used to teach, so section 5 follows Learn. Blocks run +once this helper exists: + +```bash +export SQL_SERVER=".database.windows.net" SQL_DB="" SQL_USER="" +# SQL_PASSWORD comes from your secret store, never from a file in the repository +q() { + sqlcmd -S "$SQL_SERVER" -d "${2:-$SQL_DB}" -U "$SQL_USER" -P "$SQL_PASSWORD" -C -b -I -W -Q "$1" +} +``` + +`-I` is for section 5: `.nodes()` needs `QUOTED_IDENTIFIER` ON or it fails with `Msg 1934` naming +that setting rather than Extended Events. `sqlcmd` 1.10.0 has it on always, `mssql-tools` does not. + +## 1. Prove the login can see past its own session + +```bash +q "SELECT HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'VIEW DATABASE STATE') AS can_see_others, + DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS slo;" +``` + +`1` and the results below mean what they say. `NULL` means the class or permission name was +mistyped. `0` and every empty result is worthless, and **which grant fixes it depends on the second +column**, which returns `Basic`, `S0`, `S1`, `S2`, `S3`, `P1` to `P3` or `ElasticPool`. On `Basic`, +`S0`, `S1` and any `ElasticPool` database `VIEW DATABASE STATE` is not enough: it takes the server +admin, the Microsoft Entra admin, or the server role, from `master`. + +```bash +q "ALTER SERVER ROLE [##MS_ServerStateReader##] ADD MEMBER [];" master +``` + +Learn carries that split on every DMV this page uses, including +[sys.dm_exec_connections](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-connections-transact-sql#permissions), +and says the role takes minutes to take effect. Elsewhere run +`GRANT VIEW DATABASE STATE TO [];` in that database instead. Either way the chain appears. + +## 2. Prove which lock manager you are on + +```bash +q "SELECT is_optimized_locking_on, is_read_committed_snapshot_on, + is_accelerated_database_recovery_on + FROM sys.databases WHERE database_id = DB_ID();" +``` + +Expect `1 1 1`. Not a local quirk: Learn lists optimized locking as +[always enabled on Azure SQL Database](https://learn.microsoft.com/sql/relational-databases/performance/optimized-locking#availability) +along with accelerated database recovery, and read committed snapshot as on by default. A `0` here +means the classic lock manager's names apply and every wait name on this page does not. + +## 3. Find the block without assuming the blocker has a request row + +```bash +q "SELECT r.session_id AS blocked, r.blocking_session_id AS blocker, r.wait_type, r.wait_resource, + CASE WHEN EXISTS (SELECT 1 FROM sys.dm_exec_requests AS r2 + WHERE r2.session_id = r.blocking_session_id) + THEN 'active request' ELSE 'idle, open transaction' END AS blocker_state + FROM sys.dm_exec_requests AS r WHERE r.blocking_session_id > 0;" +``` + +Whatever appears as `blocker` and never as `blocked` is the head blocker. Do not anchor a recursive +walk on "rows where `blocking_session_id = 0`": against a measured three-session chain that returned +zero rows, because the sleeping head blocker had no request row to anchor on. + +## 4. Name the head blocker and what it last ran + +An idle head blocker has no current statement, so take its text from the connection: + +```bash +q "SELECT s.session_id, s.status, s.open_transaction_count, s.last_request_end_time, t.text + FROM sys.dm_exec_sessions AS s + JOIN sys.dm_exec_connections AS c ON c.session_id = s.session_id + CROSS APPLY sys.dm_exec_sql_text(c.most_recent_sql_handle) AS t + WHERE s.is_user_process = 1 + AND EXISTS (SELECT 1 FROM sys.dm_tran_session_transactions AS x + WHERE x.session_id = s.session_id) + AND NOT EXISTS (SELECT 1 FROM sys.dm_exec_requests AS r + WHERE r.session_id = s.session_id);" +``` + +`status = sleeping` with `open_transaction_count > 0` is an application holding a transaction open +between statements: a head blocker with no row at all in `sys.dm_exec_requests`. + +## 5. Read a completed deadlock, out of the session that captured it + +Sessions are database-scoped, so they are found through the database-scoped views, in the user +database, never `master`: + +```bash +q "SELECT name FROM sys.dm_xe_database_sessions;" +``` + +Zero rows is an answer, not a failure: nothing is capturing, the deadlock that already happened is +gone, and the next one will be too. capture-with-extended-events builds the session, on Learn's +event `sqlserver.database_xml_deadlock_report`. With a ring buffer session named `deadlocks` +running, this is Learn's own read of it: + +```bash +q "DECLARE @tracename sysname = N'deadlocks'; + WITH ring_buffer AS ( + SELECT CAST(t.target_data AS XML) AS rb + FROM sys.dm_xe_database_sessions AS s + JOIN sys.dm_xe_database_session_targets AS t + ON CAST(t.event_session_address AS BINARY(8)) = CAST(s.address AS BINARY(8)) + WHERE s.name = @tracename AND t.target_name = N'ring_buffer') + SELECT d.evtdata.query('.') AS deadlock_report + FROM ring_buffer + CROSS APPLY rb.nodes('/RingBufferTarget/event[@name=''database_xml_deadlock_report'']') AS d(evtdata);" +``` + +`sqlserver.xml_deadlock_report` is a different, server-scoped event that does not exist on Azure SQL +Database. Name whichever you mean exactly. + +**Read the graph as a pair.** `` names the loser, `` carries both +processes and their ``, and `` an `` per contested resource with +its `` and ``. The fix follows from the pair: two objects touched in +opposite order, fixed by ordering them, or a range with no narrow index, fixed by adding one. + +## 6. Reproduce a block on demand, to test any of the above + +```bash +q "CREATE TABLE dbo.blk (id int PRIMARY KEY, v int); INSERT dbo.blk VALUES (1,10),(2,20);" +q "BEGIN TRAN; UPDATE dbo.blk SET v = v + 1 WHERE id = 1; WAITFOR DELAY '00:00:20'; ROLLBACK;" & +sleep 2 +q "BEGIN TRAN; UPDATE dbo.blk SET v = v + 5 WHERE id = 1; ROLLBACK;" & +sleep 2 +q "SELECT * FROM dbo.blk WHERE id = 1;" +q "SELECT * FROM dbo.blk WITH (READCOMMITTEDLOCK) WHERE id = 1;" +``` + +The first `SELECT` returns `1 10` at once: read committed snapshot versions it, so it neither blocks +nor sees the uncommitted `11`. The second waits, because `READCOMMITTEDLOCK` is Learn's documented +way to force blocking back on. Then `DROP TABLE dbo.blk;`. + +## Check it worked + +Run all three. Each has one stated answer, and a diagnosis holds only when all three do. + +```bash +q "SELECT HAS_PERMS_BY_NAME(DB_NAME(),'DATABASE','VIEW DATABASE STATE') AS grant_ok, + DATABASEPROPERTYEX(DB_NAME(),'ServiceObjective') AS slo;" +q "SELECT COUNT(*) AS capture_running FROM sys.dm_xe_database_sessions;" +q "SELECT resource_type, request_mode, request_status FROM sys.dm_tran_locks + WHERE request_session_id <> @@SPID AND resource_type IN ('PAGE','RID','KEY','XACT');" +``` + +- `grant_ok = 1`, so "nothing is blocked" means nothing is blocked, not "I cannot see it". On + `Basic`, `S0`, `S1` or `ElasticPool` that `1` alone is not enough; section 1's role settles it. +- `capture_running` above `0`, so "no deadlock recorded" is about your workload. `0` means nothing + was capturing and the answer is about your tooling. +- Run during section 6, the third returns `XACT`, mode `X`, status `GRANT`. A `KEY` row instead + means optimized locking is off and every wait name above is the wrong one. + +## Do not + +- Do not report "nothing is blocked" from a single-row result. It succeeds and looks complete + whether or not the login can see past itself. +- Do not hunt for a `system_health` session, and do not reconnect to `master` to find one. Learn + states there is no built-in one here and that sessions are database-scoped, so `Msg 208` from + `sys.dm_xe_sessions` is the correct answer, not a context bug, and no grant moves it. The + container carries both, which is how this page got it wrong until 2026-09-03. +- Do not read deadlocks out of the built-in `dl` session. Learn says not to: + `sys.fn_xe_file_target_read_file()` over many collected deadlock events can raise an + out-of-memory error in `master`, disrupt login processing, and take the application down. +- Do not predict that a plain `SELECT` blocks behind an uncommitted write, and do not read a wait + type against the classic lock manager's names alone. `LCK_M_S_XACT_MODIFY` against an `XACT` + resource is optimized locking working as designed. +- Do not conclude a deadlock report names only the victim. `` carries both, measured, + and a fix that changes only the victim usually changes nothing. + +## References + +- [Measured runs: the permission gap, the idle chain, the graph, and the container-only master result](references/blocking-visibility-and-deadlock-graphs.md): open it before disputing a wait type or either empty result, or to rerun the sequence that produced one. +- [Optimized locking](https://learn.microsoft.com/sql/relational-databases/performance/optimized-locking): + open it when a lock or wait shape here does not match what you see, for the full wait-type list. diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/references/blocking-visibility-and-deadlock-graphs.md b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/references/blocking-visibility-and-deadlock-graphs.md new file mode 100644 index 0000000..b2bddf5 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/references/blocking-visibility-and-deadlock-graphs.md @@ -0,0 +1,237 @@ +# Blocking visibility and deadlock graphs on Azure SQL Database: what was measured + +## Contents + +- [How this was measured](#how-this-was-measured) +- [The blocking-chain gap: a real permission artifact](#the-blocking-chain-gap-a-real-permission-artifact) +- [The system_health result was the container](#the-system_health-result-was-the-container) +- [The idle head blocker, measured](#the-idle-head-blocker-measured) +- [The induced deadlock](#the-induced-deadlock) +- [The deadlock graph](#the-deadlock-graph) +- [Optimized locking changes the wait type](#optimized-locking-changes-the-wait-type) +- [Read committed snapshot changes who blocks whom](#read-committed-snapshot-changes-who-blocks-whom) +- [What Microsoft Learn says](#what-microsoft-learn-says) + +## How this was measured + +**The local Azure SQL Database container**, reporting `SERVERPROPERTY('EngineEdition')` = 5 and +`SERVERPROPERTY('Edition')` = `SQL Azure`, run 2026-08-29. Naming the substrate matters here: one +result below turned out to be a container property presented as cloud behaviour for four days. A dedicated database held every test, with two accounts: + +- an administrative login, equivalent to the Azure SQL Database server admin +- an ordinary login in `db_datareader` and `db_datawriter` on the test database and nothing else, + matching an application connection string + +Two variables were moved separately: **which login** ran the query, and **which database** it was +connected to. The first section below moves only the login, the second only the database. +Concurrent sessions were separate client processes, not a simulation inside one script. + +## The blocking-chain gap: a real permission artifact + +Every row below is a separate command actually run, in order, and the only thing changing across +these rows is **which login** ran the query. + +| Step | Command | Result | +|---|---|---| +| 1 | Ordinary login: `SELECT session_id, blocking_session_id FROM sys.dm_exec_requests` while a genuine block was running elsewhere between two other logins | Returned only its own session, `blocking_session_id = 0`. No error, no hint that anything else was blocked | +| 2 | Admin: `GRANT VIEW DATABASE STATE TO ` on the test database | Succeeded | +| 3 | Same ordinary login, same query, same live block | Now returned the blocked session with the correct `blocking_session_id`, `wait_type` and `status` | +| 4 | A third, previously untouched login queried `sys.dm_os_waiting_tasks` and `sys.dm_tran_locks` during the same live block, with `VIEW DATABASE STATE` already granted | Both returned every session's rows, not just its own | + +Step 1 is the dangerous one, because it is not an error. The query runs, returns a well-formed +single row, and gives no sign that three other sessions are locked together. `VIEW DATABASE STATE` +is the only thing that changed between steps 1 and 3. Note the service objective this did **not** +cover: see the tier split in the Learn section below. + +## The system_health result was the container + +Every row below holds the login fixed and changes only **which database it is connected to**. + +| Step | Command | Result | +|---|---|---| +| 1 | Ordinary login, in the test database: `SELECT name FROM sys.dm_xe_sessions` | `Msg 208, Invalid object name 'sys.dm_xe_sessions'` | +| 2 | Same, `SELECT name FROM sys.server_event_sessions` | `Msg 208, Invalid object name` | +| 3 | Admin: `GRANT VIEW SERVER STATE TO `, confirmed in `sys.server_permissions` | Succeeded | +| 4 | Same ordinary login, still in the test database, step 1 repeated | Still `Msg 208`, despite the grant | +| 5 | Same login, reconnected to `master`, step 1 repeated | `hkenginexesession`, `system_health`, `sp_server_diagnostics session` | +| 6 | Admin login, in the test database | `Msg 208`, same as the ordinary login | + +**Read step 5 as evidence about the container and nothing else.** Steps 1, 2, 4 and 6 generalize: +the server-scoped Extended Events views do not resolve in a user database. Step 3 does not, and is +itself a container tell, because Learn states `VIEW SERVER STATE` cannot be granted in Azure SQL +Database at all. Step 5 is the one that misled this skill until 2026-09-03: the container ships a +real `master` carrying a real `system_health` session and Azure SQL Database does not, so +reconnecting to `master` is not a fix a cloud reader can apply. + +## The idle head blocker, measured + +An open transaction that has issued its statement and is now waiting on the application, not on the +engine, produces a session with no row at all in `sys.dm_exec_requests`: + +```text +session_id status open_transaction_count active_request_session_id +77 sleeping 1 NULL +``` + +A second session blocked behind it shows the correct `blocking_session_id`: + +```text +session_id status blocking_session_id wait_type command +78 suspended 77 LCK_M_S_XACT_MODIFY UPDATE +``` + +A three-level chain was also measured (79 blocked by 78, 78 blocked by idle session 77). A recursive +walk anchored on `WHERE blocking_session_id = 0 OR blocking_session_id IS NULL`, meaning "start from +whoever is not blocked," never finds 77 at all, because 77 has no row to anchor on. Against this +exact chain that form returned **zero rows**, chain and all. The corrected form in section 3 of +SKILL.md walks every `blocking_session_id` present instead, and flags per blocker whether it has a +request row of its own. It returned both blocked sessions and correctly labelled 77, named only as +somebody else's `blocking_session_id`, as idle with an open transaction. + +## The induced deadlock + +Two sessions, opposite update order, with a `WAITFOR` to make the race deterministic: + +```sql +-- Session A +BEGIN TRAN; +UPDATE dbo.accounts SET balance = balance - 10 WHERE id = 1; +WAITFOR DELAY '00:00:05'; +UPDATE dbo.accounts SET balance = balance + 10 WHERE id = 2; +COMMIT; + +-- Session B, started about a second later +BEGIN TRAN; +UPDATE dbo.accounts SET balance = balance - 10 WHERE id = 2; +WAITFOR DELAY '00:00:05'; +UPDATE dbo.accounts SET balance = balance + 10 WHERE id = 1; +COMMIT; +``` + +Session A committed. Session B's caller received: + +```text +Msg 1205, Level 13, State 72, Server SQL Azure, Line 7 +Transaction (Process ID 78) was deadlocked on lock resources with another process and has been +chosen as the deadlock victim. Rerun the transaction. +``` + +The state code was 72 in this run. Treat it as informational, not something to match on: it varies +with build and lock manager. `Msg 1205` and the message text are the stable part. + +## The deadlock graph + +The graph below was read out of the container's `system_health` ring buffer from `master`, which is +a route Azure SQL Database does not have. **The route did not generalize; the graph's shape did.** +Section 5 of SKILL.md therefore reads the same anatomy out of a database-scoped session on +`sqlserver.database_xml_deadlock_report`, which is what Learn documents for the cloud service. + +Either query calls `.nodes()`, which requires `QUOTED_IDENTIFIER` ON; without it the query fails +with `Msg 1934` naming `QUOTED_IDENTIFIER` rather than anything about permissions or Extended +Events. Which client you use decides whether you notice. Checked against local help output on +2026-09-03, `sqlcmd` 1.10.0 (go-sqlcmd) documents `-I, --enable-quoted-identifiers` as "provided for +backward compatibility. Quoted identifiers are always enabled", so it does not reproduce there; the +older `mssql-tools` `sqlcmd` does not set it. Pass `-I` and the snippet is portable across both. + +The event returned within the same second as the deadlock (deadlock at `08:27:02.959`, event +timestamp `08:27:02.959Z`). The report's `` element carried: + +- a `` naming the losing process +- a `` with **two** `` entries, the victim and the survivor, each with its + `spid`, `loginname`, `hostname`, `isolationlevel`, `lasttranstarted`, and the text of the batch it + was running in `` +- a `` with an `` entry per contested key, each carrying an `` + and a `` naming which process held the lock and which was waiting + +So the graph names the blocker, not only the victim. The raw event is not reprinted: its stack-frame +addresses alone run past a thousand characters. + +**Name this event precisely.** What was read here is `sqlserver.xml_deadlock_report`, the +server-scoped event the container's `system_health` session fires. +`sqlserver.database_xml_deadlock_report` is a different, database-scoped event, and it is the one +Azure SQL Database has. capture-with-extended-events measured that one capturing zero events across +two confirmed deadlocks on the container, and states that gap is unreproduced against the cloud +service. Two similarly named events, not two measurements of one; do not cite either as evidence +for the other. + +## Optimized locking changes the wait type + +All three flags were `1` with no opt-in step taken. Re-measured 2026-09-02 across four consecutive +fresh databases: `1/1/1` every time. A fifth run reported read committed snapshot off, did not +reproduce, and should be read as transient. Re-run before concluding the default has changed. + +A plain writer-vs-writer block, no deadlock, produced this wait, not the classic `LCK_M_U` or +`LCK_M_X` against a `KEY:` resource: + +```text +session_id status blocking_session_id wait_type +78 suspended 77 LCK_M_S_XACT_MODIFY +``` + +and in `sys.dm_os_waiting_tasks`: + +```text +resource_description: xactlock xdesIdLow=996 xdesIdHigh=0 dbid=7 id=lockc80746800 mode=X + UnderlyingResource keylock hobtid=72057594047627264 dbid=7 +``` + +and in `sys.dm_tran_locks`: + +```text +request_session_id resource_type request_mode request_status +78 XACT X GRANT +``` + +Advice written against the classic lock manager looks for a `KEY` resource type and a `U` or `X` +request mode, and will not recognize this as the same block. It is the same block, described in +terms of the transaction ID that owns it, with the underlying key as a secondary detail. + +## Read committed snapshot changes who blocks whom + +Session C updated `id = 1` and held the transaction open; session D's plain `SELECT ... WHERE id = 1` +issued while C was open returned the value from before C started, immediately, with no wait. Not a +dirty read and not a blocked read: a versioned one. Only a second **writer** against the same row +blocked, as shown above. An agent expecting the classic default will predict blocking this database +does not produce, and will also expect the in-flight value rather than the pre-transaction one. + +## What Microsoft Learn says + +Read 2026-09-03, cited rather than copied. These settle whether the measurements above generalize +past this one engine. + +- **Optimized locking is always on in the cloud, not only on the container build.** + [Optimized locking, Availability](https://learn.microsoft.com/sql/relational-databases/performance/optimized-locking#availability) + lists Azure SQL Database as "Yes (always enabled)", and the same page states accelerated database + recovery is always enabled there and read committed snapshot on by default. So the `1/1/1` + measured locally is the cloud's behaviour too and the `_XACT_` wait shapes are not a local + artifact. A self-hosted engine is the opposite case, where it is available but off by default, + which is why advice written for one does not carry over. +- **The three wait types, named exactly.** The same page lists `LCK_M_S_XACT_READ`, + `LCK_M_S_XACT_MODIFY` and `LCK_M_S_XACT`, `XACT` resources in `sys.dm_tran_locks` and + `sys.dm_exec_requests`, and an `` per resource in the report's ``. Only + `LCK_M_S_XACT_MODIFY` was reproduced here; the other two are documented, not induced. +- **`READCOMMITTEDLOCK` is the documented way to force blocking back on.** Same page, FAQ. +- **`VIEW DATABASE STATE` cannot be granted in `master`.** + [sys.dm_exec_sessions, Permissions](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-sessions-transact-sql#permissions) + states it plainly, so the two gaps in this file cannot share a fix even by accident. The same + page's example C is the documented idiom for idle sessions holding open transactions, which + section 4 of SKILL.md uses. +- **There is no built-in `system_health` session on Azure SQL Database. This file said the opposite + until 2026-09-03.** + [Use the system_health session](https://learn.microsoft.com/sql/relational-databases/extended-events/use-the-system-health-session#the-system_health-session-in-azure-sql) + states it plainly, and Azure SQL Database is not in that page's applies-to line. + [Extended Events in Azure SQL](https://learn.microsoft.com/azure/azure-sql/database/xevent-db-diff-from-svr) + gives the reason, that sessions there are always database-scoped, created `ON DATABASE` and read + through `sys.dm_xe_database_sessions`, and warns not to read deadlocks out of the built-in `dl` + session because `sys.fn_xe_file_target_read_file()` over many events can raise an out-of-memory + error in `master`. The documented cloud route is a database-scoped session on + `sqlserver.database_xml_deadlock_report`: + [Analyze and prevent deadlocks](https://learn.microsoft.com/azure/azure-sql/database/analyze-prevent-deadlocks#collect-deadlock-graphs-in-azure-sql-database-with-extended-events). +- **The tier split on `VIEW DATABASE STATE`,** carried on every DMV this skill uses, including + [sys.dm_exec_connections](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-connections-transact-sql#permissions): + on Basic, S0, S1 and elastic pool databases it takes the server admin, the Microsoft Entra admin, + or `##MS_ServerStateReader##`; everywhere else `VIEW DATABASE STATE` is enough. + [Monitor performance using DMVs](https://learn.microsoft.com/azure/azure-sql/database/monitoring-with-dmvs#permissions) + gives the grant, `ALTER SERVER ROLE [##MS_ServerStateReader##] ADD MEMBER [login];` in `master`, + and notes it takes minutes. The 2026-08-29 run was administrative throughout and never crossed + this boundary, so this one is cited, not measured. diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/skill.spec.jsonc new file mode 100644 index 0000000..91226f2 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-blocking-and-deadlocks/skill.spec.jsonc @@ -0,0 +1,193 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "diagnose-blocking-and-deadlocks", + "domain": "performance-and-diagnostics", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked to diagnose blocking or read a deadlock on Azure SQL Database, an agent treats two different diagnostic failures as the same kind of problem and gets both wrong. The first is a genuine permission artifact: an ordinary login with no VIEW DATABASE STATE grant sees only its own session in sys.dm_exec_requests, sys.dm_os_waiting_tasks and sys.dm_tran_locks and misses a real blocking chain running between two other logins, with no error at all, and granting that one database-scoped permission to the same login, with no other change, made the whole chain visible. The second looks identical from the outside but is not a permission problem at all: sys.dm_xe_sessions, sys.server_event_sessions and sys.dm_xe_session_targets raise Invalid object name (Msg 208) in a user database, and GRANT VIEW SERVER STATE changed nothing, because there is no server-scoped Extended Events surface to grant past. An agent reads that Msg 208 as a context bug and goes looking for a system_health session to read a deadlock out of. Microsoft Learn states there is no built-in system_health session in Azure SQL Database and that event sessions there are always database-scoped, so unless somebody created a session on sqlserver.database_xml_deadlock_report before the deadlock happened, the graph is gone and no grant, no reconnect and no master will recover it. The local container does carry a master with a system_health session, which is exactly how this skill shipped the wrong answer between 2026-08-29 and 2026-09-03: the run was correct and the substrate was never named. A second correction rides along: on Basic, S0, S1 and elastic pool databases, VIEW DATABASE STATE is not the grant that makes the chain visible either, and ##MS_ServerStateReader## is.", + "value_probe": { + "gate": "The answer separates the two failures instead of treating them as one problem: the empty blocking result is what the connected login is permitted to see and is fixed by granting the right database scoped permission to it, while the missing graph from yesterday is neither a permission nor a connection context problem and cannot be recovered, so the remedy is a capture created before the next one happens.", + "task": "Our checkout runs against an Azure SQL Database in the cloud, a General Purpose database and not the local container. It hangs for about thirty seconds under load and then some requests fail. I connected with the application's own login and ran:\n\n```sql\nSELECT session_id, blocking_session_id, wait_type, wait_time\nFROM sys.dm_exec_requests\nWHERE blocking_session_id <> 0;\n```\n\nIt returns nothing at all, every time, while the application is visibly stuck. sys.dm_tran_locks only ever shows my own session and sys.dm_os_waiting_tasks comes back empty too.\n\nWe also had two deadlock errors in the application log yesterday at 14:05 and I want to see what they were fighting over. A colleague told me to read the graph out of the built in diagnostic session, but sys.dm_xe_sessions comes back as an invalid object name, so I assumed I was connected to the wrong database and I have not got any further.\n\nWhat do I do, for both of these?", + "states": [ + "A login without the relevant database scoped state permission sees only its own sessions in those views, and is told nothing about the rows it cannot see.", + "The empty result is therefore not evidence that no blocking is happening.", + "The invalid object name is not a connection context problem and is not fixed by connecting somewhere else or by granting a server scoped permission.", + "A deadlock graph for something that has already happened is gone unless a capture was already running when it happened.", + "Catching the next occurrence means creating a database scoped capture in advance of it." + ], + "refutes": [ + "The invalid object name for the Extended Events views means the query was run against the wrong database, and running it from master will return the sessions.", + "Azure SQL Database has a built in, always running diagnostic session that retains recent deadlock graphs, so yesterday's graph can still be read out of it.", + "Granting VIEW SERVER STATE is what makes the Extended Events views resolve on this database.", + "Seeing only your own session in the request and lock views means nothing is blocking, so the delay must be in the application or the network.", + "The deadlock graph can be recovered after the fact from an error log, a default trace or the database's own history." + ] + }, + "posture": ["inspect"], + // Demoted from preview to draft on 2026-09-20, and this one is a trade rather + // than a clean loss, which is why the numbers are written out here. The value + // measurement of 2026-09-05, ten runs per arm, returned UNCLEAR. The skill helps + // in three places: the gate reads 10 of 10 against 6 of 10, the statement that + // catching the next occurrence means creating the capture in advance 10 of 10 + // against 5 of 10, and it removes the falsehood that a deadlock graph can be + // recovered after the fact, 0 of 10 against 6 of 10. Against that, the statement + // that a login without the database scoped state permission sees only its own + // sessions falls to 5 of 10 with the skill from 10 of 10 without, and that is the + // fact a reader most needs when the blocking view comes back empty. Carlos took + // the trade on 2026-09-20: losing that one costs more than the three gains earn. + // The three gains are real, so this skill is a rewrite candidate rather than a + // deletion candidate. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "diagnose-slow-query", + "diagnose-resource-pressure", + "capture-with-extended-events", + "read-execution-plan", + "t-sql-upserts-merge" + ], + "validation": { + "target": "both", + "setup": [ + "a live engine reachable over sqlcmd reporting SERVERPROPERTY('EngineEdition') = 5 and SERVERPROPERTY('Edition') = 'SQL Azure'", + "a database created for the check, separate from other work on the same engine", + "the ability to open at least three concurrent client sessions against that database", + "one ordinary login with only db_datareader and db_datawriter, connected once to the master database and once to a user database" + ], + "assert": [ + "a genuine blocking chain (a writer blocked behind another writer holding an open transaction) is invisible in sys.dm_exec_requests to a third-party login with no VIEW DATABASE STATE grant, and becomes visible after that single grant, except on Basic, S0, S1 and elastic pool databases where ##MS_ServerStateReader## is required instead", + "an idle session holding an open transaction (no active request) is correctly named as the blocking_session_id of the session it blocks, even though it has no row of its own in sys.dm_exec_requests", + "sys.dm_xe_sessions and sys.dm_xe_session_targets raise Invalid object name (Msg 208) in a user database and are not made to resolve by any VIEW SERVER STATE grant, while sys.dm_xe_database_sessions resolves in the same database", + "an induced deadlock produces Msg 1205 to the victim's caller, and, with a database-scoped session on sqlserver.database_xml_deadlock_report already running, a matching event readable from the user database with a single XML query over sys.dm_xe_database_session_targets", + "the deadlock graph's process-list contains both the victim and the blocking process, not the victim alone", + "a lock or wait reported under optimized locking uses an _XACT_ wait type and an XACT: resource description rather than the classic KEY: lock resource format" + ], + "probes": [ + { + "sql": "SELECT name FROM sys.dm_xe_sessions;", + "expect": { + "error": 208 + }, + "why": "The server-scoped Extended Events DMV does not resolve in a user database, measured 2026-08-29, and GRANT VIEW SERVER STATE (confirmed present in sys.server_permissions) changed nothing, because there is no permission boundary here to grant past. Msg 208 is the correct answer rather than a context bug: Microsoft Learn states event sessions on Azure SQL Database are always database-scoped. The next probe pairs with this one, showing what does resolve. Reconnecting to master returned rows on the container, and that is a container property, not this skill's advice." + }, + { + "sql": "SELECT name FROM sys.server_event_sessions;", + "expect": { + "error": 208 + }, + "why": "The catalog view fails the same way as the DMV, which matters because an agent that hits one tries the other next. Measured 2026-08-29. The conclusion the two Msg 208s support is the correct one: Microsoft Learn's Use the system_health session page states there is no built-in system_health session in Azure SQL Database, and its applies-to line covers SQL Server and Azure SQL Managed Instance only. What they do not support is reconnecting to master, which works on the container and has nothing to reach on the cloud service." + }, + { + "sql": "SELECT COUNT(*) AS started_sessions FROM sys.dm_xe_database_sessions;", + "expect": { + "succeeds": true + }, + "why": "The success the skill teaches, which the previous two probes only ever encoded as a failure. Section 5 sends the reader to the database-scoped views in the user database, so this one has to resolve where sys.dm_xe_sessions raised Msg 208 two probes ago. The count itself is deliberately unasserted: it is zero on a database nobody has built a session on, and the skill's whole point is that zero is an answer rather than an error. Microsoft Learn's Extended Events in Azure SQL page names this DMV as the database-scoped replacement." + }, + { + "sql": "SELECT DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS slo;", + "expect": { + "succeeds": true + }, + "why": "Section 1 now tells the reader to read the service objective before choosing a grant, because Microsoft Learn requires ##MS_ServerStateReader## rather than VIEW DATABASE STATE on Basic, S0, S1 and elastic pool databases, and that branch is unusable if the call does not resolve where the reader is diagnosing. This probe asserts only that it resolves. It deliberately asserts no value: Learn documents ServiceObjective as nvarchar(32) returning Basic, S0 through S3, P1 through P3, ElasticPool, System, or NULL when the database is not started, and what the container returns for it has not been measured here, so asserting one would be inventing a measurement. The tier rule itself cannot be probed on the container at all, which has no service objectives to differ between; it is cited from Learn in SKILL.md and in the reference, and named here so nobody mistakes a green run for evidence of it." + }, + { + "sql": "SELECT (SELECT COUNT(*) FROM sys.dm_os_waiting_tasks WHERE 1 = 0) AS waiting_tasks,\n (SELECT COUNT(*) FROM sys.dm_tran_locks WHERE 1 = 0) AS tran_locks;", + "expect": { + "rows": [ + [ + 0, + 0 + ] + ] + }, + "why": "The negative control for the two probes above. These database-scoped views resolve perfectly well in a user database, so an empty blocking picture here is a permission artifact that VIEW DATABASE STATE fixes, not the object-scoping artifact that only a connection to master fixes. The skill's entire correction is telling those two apart." + }, + { + "sql": "SELECT is_optimized_locking_on, is_read_committed_snapshot_on, is_accelerated_database_recovery_on\n FROM sys.databases WHERE database_id = DB_ID();", + "expect": { + "rows": [ + [ + 1, + 1, + 1 + ] + ] + }, + "why": "Measured 2026-08-29: all three were already on with no opt-in step taken. This is the precondition behind the two behaviours advice written for the classic lock manager gets wrong, an _XACT_ wait type against an XACT resource instead of KEY, and a plain SELECT that versions rather than blocks. If a future default flips, the skill's lock shapes stop matching before anything else does." + }, + { + "sql": "SELECT\n r.session_id AS blocked_session,\n r.blocking_session_id AS blocker_session,\n r.wait_type,\n r.wait_time,\n CASE WHEN EXISTS (\n SELECT 1 FROM sys.dm_exec_requests AS r2 WHERE r2.session_id = r.blocking_session_id\n )\n THEN 'blocker has an active request'\n ELSE 'blocker is idle: open transaction, no active request'\n END AS blocker_state\n FROM sys.dm_exec_requests AS r\n WHERE r.blocking_session_id > 0;", + "expect": { + "succeeds": true + }, + "why": "The corrected chain query is published in the reference file for a reader to paste, and a snippet that no longer compiles against the engine is worse than no snippet. This probe asserts nothing about a blocking chain, which needs concurrent sessions, only that every column and view the published query names still resolves on the target substrate." + }, + { + "sql": "SELECT HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'VIEW DATABASE STATE') AS can_see_others;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "Step 1 of the skill tells a reader to paste this exact call before trusting any empty blocking result, so the call itself has to be valid on this substrate. Microsoft Learn restricts securable_class on Azure SQL Database to DATABASE, OBJECT, ROLE, SCHEMA or USER, and HAS_PERMS_BY_NAME returns NULL rather than an error when the class or permission name is wrong. A NULL here would mean the skill's headline check silently answers nothing. 1 is the answer on the administrative connection a probe runs on, because VIEW DATABASE STATE is implied by CONTROL on the database." + }, + { + "sql": "CREATE TABLE dbo.probe_optlock_t0 (a int NOT NULL PRIMARY KEY, b int NULL);\nINSERT INTO dbo.probe_optlock_t0 VALUES (1, 10), (2, 20), (3, 30);\nBEGIN TRANSACTION;\nUPDATE dbo.probe_optlock_t0 SET b = b + 10;\nSELECT resource_type, request_mode, request_status\n FROM sys.dm_tran_locks\n WHERE request_session_id = @@SPID\n AND resource_type IN ('PAGE', 'RID', 'KEY', 'XACT');\nCOMMIT TRANSACTION;", + "expect": { + "rows": [ + [ + "XACT", + "X", + "GRANT" + ] + ] + }, + "why": "The lock shape the skill exists to correct, made executable in one session rather than asserted from the 1/1/1 flags. This is Microsoft Learn's own transaction ID locking example on the optimized locking page, which states that with optimized locking on, a three-row update holds a single X lock on the XACT resource and no KEY or PAGE locks, and four locks including three X KEY locks when it is off. Measured here 2026-08-29 as XACT / X / GRANT during a live block. A KEY row instead is the one result that invalidates every wait name on the skill page at once." + }, + { + "cmd": "sqlcmd '-?' | tr -s '[:space:]' ' ' | grep -q 'Quoted identifiers are always enabled'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The deadlock query needs QUOTED_IDENTIFIER ON, and the skill claims the client decides whether you ever notice. Checked against local help output on 2026-09-03: sqlcmd 1.10.0 (go-sqlcmd) documents -I as provided for backward compatibility because quoted identifiers are always enabled, while the older mssql-tools sqlcmd does not set it. host, not container: this is the client a developer installs, and the wording being asserted is that build's. The tr is load-bearing, not decoration. This probe exited 1 on every run before 2026-09-03 because sqlcmd wraps its help text and the asserted phrase spans two lines, so a line-oriented grep could never match it. The claim was right and the matcher was wrong, so the matcher was fixed; flattening whitespace first makes the assertion test the sentence rather than the column width. If a future sqlcmd drops the line, the skill's portability note needs rewriting rather than quietly rotting." + } + , + { + "sql": "CREATE TABLE dbo.probe_b_qi (id int NOT NULL PRIMARY KEY, c AS (id * 2));\nCREATE UNIQUE INDEX probe_b_qi_ix ON dbo.probe_b_qi (c);\nGO\nSET QUOTED_IDENTIFIER OFF;\nINSERT INTO dbo.probe_b_qi (id) VALUES (1);", + "expect": { + "error": 1934 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A write against a table carrying an indexed computed column needs QUOTED_IDENTIFIER ON, and the message names indexed views and filtered indexes rather than the setting. A session that lacks it fails on write and looks like a blocking problem to whoever is reading the symptom. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Why is this transaction blocking everything on our Azure SQL database", + "Read this deadlock graph and tell me which session to fix", + "Find out who is blocking session 78 right now", + "We keep getting Msg 1205 was deadlocked on lock resources with another process, help me capture the deadlock" + ], + "implicit": [ + "our checkout page is hanging under load and I ran a blocking query but nothing shows blocked", + "blocking_session_id is always 0 in sys.dm_exec_requests but the app is clearly stuck waiting", + "who is the head blocker and what is it running right now", + "a session has been idle for ten minutes but still shows an open transaction and I cannot find what it is running" + ], + "negative": [ + "Why is this single query slow even though nothing else is running against the database", + "The database is throttled and CPU is pegged at 100 percent, should we scale up", + "Build me an Extended Events session to capture query text over the next hour", + "Read this execution plan and tell me why it chose a table scan", + "Write an upsert that is safe when two sessions run it at the same time" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure deleted file mode 120000 index e8c542b..0000000 --- a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure +++ /dev/null @@ -1 +0,0 @@ -../../../skills/diagnose-resource-pressure \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/SKILL.md b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/SKILL.md new file mode 100644 index 0000000..1856232 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/SKILL.md @@ -0,0 +1,192 @@ +--- +name: diagnose-resource-pressure +description: >- + Answers whether an Azure SQL Database is slow because of CPU, data or log IO, memory, or a + worker and session limit. Use when someone reports the database as slow, throttled or timing + out under load, pastes a resource governance error such as "the request limit for the database + is 200 and has been reached" or a raw error number 10928, 10929 or 10936, asks whether to scale + up the service tier, or is reading sys.dm_db_resource_stats, sys.dm_os_performance_counters or + sys.dm_os_wait_stats. Covers the local Azure SQL Database container too. Does not read an + execution plan (read-execution-plan), diagnose a blocking chain + (diagnose-blocking-and-deadlocks), rewrite a slow query (diagnose-slow-query), resolve a + connection failure (diagnose-connection-errors), size a connection pool (connect-to-azure-sql) + or speed up a bulk load (bulk-load-and-bulk-copy). +--- + +# Diagnose Azure SQL Database resource pressure + +**On SQL Server you look at the machine. Here the machine is not yours and the governed limit is +the product.** Every headline number is a percentage of a ceiling set by the service objective, +not a reading off hardware. Ask which limit is being hit, not how busy the box is: CPU, +data IO, log IO, memory, and workers and sessions each have their own fix, and only some are +fixed by scaling. + +Checked 2026-09-03 against Microsoft Learn and 2026-08-29 against a live engine reporting +`SERVERPROPERTY('EngineEdition')` = 5 and `Edition` = `SQL Azure`, the local Azure SQL Database +container. + +## 1. Substrate and purchasing model + +```sql +SELECT SERVERPROPERTY('EngineEdition') AS engine_edition, SERVERPROPERTY('Edition') AS edition; +SELECT TOP 1 end_time, dtu_limit, cpu_limit +FROM sys.dm_db_resource_stats ORDER BY end_time DESC; +``` + +`dtu_limit` is NULL on a vCore database and `cpu_limit`, the vCore count, is NULL on a DTU one, so +this row settles the purchasing model. On a DTU database +`SELECT COUNT(*) FROM sys.dm_os_schedulers WHERE status = N'VISIBLE ONLINE';` is the only way to +see a vCore count, and on the container that count is whatever CPUs it was given and says nothing +about a tier. The row settles the substrate too: locally the second statement fails +`Msg 208, Invalid object name 'sys.dm_db_resource_stats'`, that view being Azure SQL Database +only. Both answers steer everything below, so run it first. + +## 2. CPU + +```sql +SELECT TOP 20 end_time, avg_cpu_percent, avg_instance_cpu_percent, avg_memory_usage_percent +FROM sys.dm_db_resource_stats ORDER BY end_time DESC; +``` + +`avg_cpu_percent` is the user workload against the user CPU limit; `avg_instance_cpu_percent` is +every workload, internal included, against another limit. Learn is explicit: different +scales, not comparable, and either in the 70 to 100 range flattens throughput. On a DTU +database the headline is +`avg_dtu_percent = MAX(avg_cpu_percent, avg_data_io_percent, avg_log_write_percent)`. + +The container has no such summary. Order the plan cache on the metric itself, never on a +worker-time-to-elapsed ratio, which silently drops the queries you want: + +```sql +SELECT TOP 20 qs.total_worker_time, qs.execution_count, + qs.total_worker_time / qs.execution_count AS avg_worker_time, + SUBSTRING(st.text, 1, 200) AS query_text +FROM sys.dm_exec_query_stats AS qs +CROSS APPLY sys.dm_exec_sql_text(qs.sql_handle) AS st +ORDER BY qs.total_worker_time DESC; +``` + +## 3. Data IO and log IO + +```sql +SELECT file_id, num_of_reads, num_of_writes, io_stall_read_ms, io_stall_write_ms, + io_stall_queued_read_ms, io_stall_queued_write_ms +FROM sys.dm_io_virtual_file_stats(DB_ID(), NULL); +``` + +Runs in both places. The two queued columns separate delay added by IO governance from delay added +by storage; `avg_data_io_percent` and `avg_log_write_percent` are the governed view on top. + +Log rate governance shows up as named waits, and only in the cloud: + +```sql +SELECT wait_type, waiting_tasks_count, wait_time_ms FROM sys.dm_os_wait_stats +WHERE wait_type IN ('LOG_RATE_GOVERNOR', 'POOL_LOG_RATE_GOVERNOR', 'INSTANCE_LOG_RATE_GOVERNOR') + AND waiting_tasks_count > 0; +``` + +The container has no log rate governor, so this returns nothing there however hard it is pushed, +and a saturated container log surfaces as `WRITELOG`. Empty locally says nothing about the cloud. + +## 4. Memory + +```sql +SELECT TOP 5 type, pages_kb FROM sys.dm_os_memory_clerks ORDER BY pages_kb DESC; +SELECT resource_semaphore_id, available_memory_kb, grantee_count, waiter_count +FROM sys.dm_exec_query_resource_semaphores; +``` + +Both run in both places; `waiter_count` above zero with `available_memory_kb` near zero is +grants queuing. Do not read `avg_memory_usage_percent` near 100 as a fault: Learn states the +engine uses all available memory for its data cache whatever the load, which is why the DTU +formula excludes it. + +`sys.dm_os_performance_counters` exists on Azure SQL Database, but its permission is not uniform: +Basic, S0, S1 and any pooled database need the server admin, the Microsoft Entra admin or +`##MS_ServerStateReader##`, elsewhere `VIEW DATABASE STATE` is enough. The counter query that works +on S3 fails on S1 as a permission error, not a missing counter. + +## 5. Workers and sessions: 10928, 10929, 10936 + +Read the Resource ID first: `1` is workers, `2` is sessions, not one problem. + +```sql +SELECT TOP 20 end_time, max_worker_percent, max_session_percent +FROM sys.dm_db_resource_stats ORDER BY end_time DESC; +SELECT r.session_id, r.blocking_session_id, r.wait_type, r.status +FROM sys.dm_exec_requests AS r WHERE r.session_id > 50; +``` + +Learn keeps "request limit" in 10928 and 10936 for backward compatibility only, from +when a request was one worker. What ran out is workers. A worker is spent by a login, a serial +query, and every parallel thread above MAXDOP 1, so a blocking pileup or parallelism raises the +count far faster than connection volume, and shrinking the pool alone does not fix it. S3 +allows 200 concurrent workers and 2400 sessions, so "the request limit for the database is 200 and +has been reached" is an S3 out of workers. + +While the limit is being hit, ordinary connections are refused. One still gets in, and is the only +way to run the query above during the incident: + +```bash +sqlcmd -S your-server.database.windows.net -d -U \ + -P "$SQLCMD_PASSWORD" -A -Q "SELECT session_id, blocking_session_id, wait_type \ + FROM sys.dm_exec_requests WHERE blocking_session_id <> 0;" +``` + +`-A` opens the diagnostic connection, also reachable by prefixing the server name with `admin:`. +It needs a logical server administrator, is not supported with `-G`, and only one exists per +database. + +## 6. MAXDOP, the worker multiplier + +```sql +SELECT [value], value_for_secondary FROM sys.database_scoped_configurations WHERE [name] = 'MAXDOP'; +``` + +Since September 2020 a new Azure SQL Database reports `8` here, and this view is the authority: +nothing is applied underneath. `0` in the cloud means the database predates that change and +uses every logical processor up to 64, which Learn recommends against; `0` on the container is +just the engine default. `sp_configure` is the surface in neither place, and it is absent rather +than restricted: Learn's `sys.sp_configure` page does not list Azure SQL Database, and on the +container `EXEC sp_configure 'max degree of parallelism';` returns `Msg 2812, Could not find stored +procedure 'sp_configure'`, not the option error `Msg 15123` that sends a reader hunting a +permission problem. Change it with +`ALTER DATABASE SCOPED CONFIGURATION SET MAXDOP = 8;`, which needs server admin, `db_owner` or +`ALTER ANY DATABASE SCOPED CONFIGURATION` and cannot run against `master`. + +## 7. Only now, the tier + +Scaling raises CPU, memory and IO headroom and the worker and session ceilings. It does not fix a +query doing needless work, and it does not change MAXDOP. Name the exhausted resource first. + +## Check it worked + +The diagnosis is finished when this returns a row and you can name the column that was high: + +```sql +SELECT TOP 1 end_time, avg_cpu_percent, avg_instance_cpu_percent, avg_data_io_percent, + avg_log_write_percent, max_worker_percent, max_session_percent +FROM sys.dm_db_resource_stats ORDER BY end_time DESC; +``` + +Rows land every 15 seconds and reach back about an hour; an older incident needs +`sys.resource_stats` in the logical server's `master`, 5 minute granularity over 14 days. After the +fix that column falls and the others do not move. + +On the container both return `Msg 208` and nothing substitutes, so the check is the step 2 plan +cache query and the step 3 file stats run twice across the change: the heaviest +`total_worker_time` drops, or `io_stall_write_ms` does. An empty cloud-only result is not health. + +## Do not + +- Do not read `Msg 208` on the container as a broken database. `sys.dm_db_resource_stats`, + `sys.resource_stats`, `sys.dm_user_db_resource_governance` and + `sys.dm_instance_resource_governance` are not implemented there. +- Do not treat `sys.database_service_objectives` on the container as real capacity: it answers + `GP_Gen5_2` for every database and nothing enforces it. + +## References + +- Open [references/dmv-and-limits-reference.md](references/dmv-and-limits-reference.md) when a + statement above returns `Msg 208` and you need the local substitute, or before quoting a worker + or session limit for a tier. diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/references/dmv-and-limits-reference.md b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/references/dmv-and-limits-reference.md new file mode 100644 index 0000000..0fba35d --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/references/dmv-and-limits-reference.md @@ -0,0 +1,158 @@ +# DMVs, error codes and limits for Azure SQL Database resource pressure + +Checked on 2026-09-03 against Microsoft Learn and on 2026-08-29 against a live engine reporting `SERVERPROPERTY('EngineEdition')` = 5 and +`SERVERPROPERTY('Edition')` = `SQL Azure` (the local Azure SQL Database container). + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [DMV availability, container versus a real Azure SQL Database](#dmv-availability-container-versus-a-real-azure-sql-database) +- [The purchasing model, and the numbers behind the percentages](#the-purchasing-model-and-the-numbers-behind-the-percentages) +- [Resource governance error codes](#resource-governance-error-codes) +- [MAXDOP: the same 0 means two different things](#maxdop-the-same-0-means-two-different-things) +- [Worker and session limits by tier](#worker-and-session-limits-by-tier) +- [Transaction log rate governance](#transaction-log-rate-governance) +- [Sources](#sources) + +## How to use this file + +The skill body gives the decision order. This file gives what each object returned when checked +directly. Read the availability table first when the target is the container: about a third of the +objects a model reaches for do not exist there. + +## DMV availability, container versus a real Azure SQL Database + +Every "container" cell was run directly. Every "cloud" cell is Learn's documented behavior, not +measured, because no paid database was available for this check. + +| Object | Container | Azure SQL Database (documented) | +|---|---|---| +| `sys.dm_db_resource_stats` | Does not exist. `SELECT * FROM sys.dm_db_resource_stats` raises `Msg 208, Invalid object name 'sys.dm_db_resource_stats'` | Exists, per-database, one row roughly every 15 seconds, the primary answer to "is it CPU, IO or memory" | +| `sys.resource_stats` | Does not exist in any database, including `master` | Exists in the logical server's `master` database only, five-minute granularity, longer retention | +| `sys.dm_user_db_resource_governance` | Does not exist | Exists, reports the effective governance limits (including MAXDOP and storage quota) actually in force for the current database | +| `sys.dm_instance_resource_governance` | Does not exist | Exists, server-wide governance settings | +| `sys.database_service_objectives` | Exists, but returns a fixed placeholder (`GeneralPurpose` / `GP_Gen5_2`) for every database regardless of the container's real size | Exists, reflects the real provisioned or serverless service objective | +| `sys.dm_os_wait_stats` | Exists, real data (76 wait types under load in the check) | Exists, scoped to the resource pool serving the current database | +| `sys.dm_os_performance_counters` | Exists (2695 rows observed), including the full `SQLServer:Buffer Manager` and log-flush counter families, because the container is not sharing the process with any other tenant | Exists, but the permission differs by service objective: Basic, S0, S1 and any pooled database need the server admin, the Microsoft Entra admin or `##MS_ServerStateReader##`, elsewhere `VIEW DATABASE STATE` is enough. The numbers describe the shared engine process, so it is no stand-in for `sys.dm_db_resource_stats` | +| `sys.dm_exec_query_resource_semaphores`, `sys.dm_os_memory_clerks`, `sys.dm_exec_query_stats`, `sys.dm_exec_cached_plans`, `sys.dm_exec_requests`, `sys.dm_exec_sessions` | All exist and populate normally: memory grant semaphore state, `MEMORYCLERK_SQLBUFFERPOOL` as the largest clerk in the check, plan cache with per-plan CPU and IO, and live `blocking_session_id` and `wait_type` | All exist | +| `sp_configure 'max degree of parallelism'` | Not installed: `Msg 2812, Could not find stored procedure 'sp_configure'`, measured 2026-09-03 | Not supported either, and not the configuration surface; see the MAXDOP section below | +| `sys.database_scoped_configurations` (MAXDOP) | Exists, reports `value = 0` (measured) | Exists, reports `8` on every database created since September 2020, and `0` on one created before it | + +## The purchasing model, and the numbers behind the percentages + +Every percentage in `sys.dm_db_resource_stats` is against a governed limit, so the limit has to be +known first. The same row identifies the model: + +```sql +SELECT TOP 1 end_time, dtu_limit, cpu_limit, avg_cpu_percent, avg_instance_cpu_percent, + avg_data_io_percent, avg_log_write_percent, avg_memory_usage_percent, + max_worker_percent, max_session_percent +FROM sys.dm_db_resource_stats ORDER BY end_time DESC; +``` + +- `dtu_limit` is `NULL` on a vCore database; `cpu_limit`, the vCore count, is `NULL` on a DTU one. +- On a DTU database the headline is not a column. Learn defines it as + `avg_dtu_percent = MAX(avg_cpu_percent, avg_data_io_percent, avg_log_write_percent)`, memory + excluded because the data cache keeps `avg_memory_usage_percent` near 100 regardless of load. +- `avg_cpu_percent` (user workload) and `avg_instance_cpu_percent` (every workload) are + percentages of two different limits, not comparable with each other, and either in the 70 to 100 + range flattens throughput. +- `sys.dm_user_db_resource_governance` is the view Learn names as where the actual DTU and vCore + limits are exposed, and the one to compare when sizing a move between models. Cloud only. + +## Resource governance error codes + +Verified against Microsoft Learn on 2026-08-29. + +| Error | Resource ID meaning | Text (abridged) | +|---|---|---| +| `10928` | `1` = workers, `2` = sessions | `Resource ID : %d. The %s limit for the database is %d and has been reached` | +| `10936` | Same as `10928`, for an elastic pool | `Resource ID : %d. The %s limit for the elastic pool is %d and has been reached` | +| `10929` | Same Resource ID meaning, phrased as a soft limit | `Resource ID: %d. The %s minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d...` | + +Learn's note on `10928`: "request limit" is backward compatibility from when a request was one +worker. What is reached is the worker count, consumed by a login, a serial query, or each parallel +thread above MAXDOP 1. + +`40613`, `18456` and `4060` read similarly and are not resource governance at all; +`diagnose-connection-errors` owns them. + +## MAXDOP: the same `0` means two different things + +**Corrected 2026-09-03.** An earlier revision claimed Azure SQL Database reports `0` while +applying `8` invisibly underneath. Learn contradicts it: "the MAXDOP database-scoped configuration +is set to 8" for each new single or elastic pool database, and Learn's own example reads +`sys.database_scoped_configurations` to determine it. The view is the authority. + +- **`8` in the cloud** is the default for any database created since September 2020. Databases + created before that date were not changed. +- **`0` in the cloud** therefore means the database predates that change, or somebody set it back. + Learn recommends against `0`: it uses the logical processor count up to 64, so a scale-up + silently widens every parallel plan. +- **`0` on the container** is measured, and is the engine default with no control plane to set + anything else. It says nothing about what a cloud database would report. + +Check and set it explicitly rather than trusting the default silently: + +```sql +SELECT [value], value_for_secondary FROM sys.database_scoped_configurations WHERE [name] = 'MAXDOP'; +ALTER DATABASE SCOPED CONFIGURATION SET MAXDOP = 8; +``` + +`ALTER DATABASE SCOPED CONFIGURATION` requires the server admin, `db_owner`, or a principal granted +`ALTER ANY DATABASE SCOPED CONFIGURATION`, and cannot run against `master`. `sp_configure` is not +an alternative, and it is absent rather than restricted. Learn lists `sp_configure` and +`RECONFIGURE` among the syntax Azure SQL Database does not support, and the `sys.sp_configure` +Applies to list omits Azure SQL Database. On the container it is not installed: +`EXEC sp_configure 'max degree of parallelism';` returns `Msg 2812, Could not find stored procedure +'sp_configure'`, measured 2026-09-03. An earlier revision published `Msg 15123`, the option error, +which would have meant the procedure was there and had rejected the option name. + +## Worker and session limits by tier + +The maximum worker count is set by the compute size, not by a fixed platform number, so the number +in a `10928` message identifies the tier. Microsoft Learn's DTU table for single databases: + +| Compute size | Basic | S0 | S1 | S2 | S3 | S4 | S6 | S7 | +|---|---|---|---|---|---|---|---|---| +| Max concurrent workers | 30 | 60 | 90 | 120 | 200 | 400 | 800 | 1600 | +| Max concurrent sessions | 300 | 600 | 900 | 1200 | 2400 | 4800 | 9600 | 19200 | + +So "the request limit for the database is 200 and has been reached" is an S3 out of workers, with a +session ceiling twelve times higher. The vCore equivalents are per compute size and hardware +generation: fetch +[vCore resource limits](https://learn.microsoft.com/azure/azure-sql/database/resource-limits-vcore-single-databases) +rather than quoting a remembered number. + +The container enforces no such limit; `sys.dm_os_sys_info` reported `max_workers_count` of 512, a +generic engine default unrelated to any compute size and not worth quoting as a service tier. + +## Transaction log rate governance + +Documented wait types, visible in `sys.dm_exec_requests` and `sys.dm_os_wait_stats` on a real +database when log rate governance is actively throttling: + +| Wait type | Meaning | +|---|---| +| `LOG_RATE_GOVERNOR` | Database-level log rate limiting | +| `POOL_LOG_RATE_GOVERNOR` | Elastic pool-level log rate limiting | +| `INSTANCE_LOG_RATE_GOVERNOR` | Instance-level log rate limiting | +| `HADR_THROTTLE_LOG_RATE_SEND_RECV_QUEUE_SIZE` | Feedback control: a replica is not keeping up | +| `HADR_THROTTLE_LOG_RATE_LOG_SIZE` | Feedback control: avoiding an out-of-log-space condition | +| `HADR_THROTTLE_LOG_RATE_MISMATCHED_SLO` | Geo-replication feedback control | + +None appeared in the container's `sys.dm_os_wait_stats` during the check: it has no log rate +governance to throttle against, so a bulk load that saturates its log shows as raw log-flush wait +time (`WRITELOG`, or elevated `Log Flush Wait Time` in `sys.dm_os_performance_counters`) instead. + +## Sources + +- [Resource management](https://learn.microsoft.com/azure/azure-sql/database/resource-limits-logical-server), fetched 2026-08-29. +- [Configure the max degree of parallelism (MAXDOP)](https://learn.microsoft.com/azure/azure-sql/database/configure-max-degree-of-parallelism), fetched 2026-09-03. +- [DTU purchasing model](https://learn.microsoft.com/azure/azure-sql/database/service-tiers-dtu) and [DTU resource limits](https://learn.microsoft.com/azure/azure-sql/database/resource-limits-dtu-single-databases), fetched 2026-09-03: the avg_dtu_percent formula and the worker and session table. +- [sys.dm_os_performance_counters](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-os-performance-counters-transact-sql), fetched 2026-09-03: the per-service-objective permission split. +- [sys.sp_configure](https://learn.microsoft.com/sql/relational-databases/system-stored-procedures/sp-configure-transact-sql) and [T-SQL differences](https://learn.microsoft.com/azure/azure-sql/database/transact-sql-tsql-differences-sql-server), fetched 2026-09-03: the Applies to list, and `sp_configure` as unsupported. +- [sqlcmd utility](https://learn.microsoft.com/sql/tools/sqlcmd/sqlcmd-utility), fetched 2026-09-03: the -A diagnostic connection and its incompatibility with -G. +- [Troubleshoot common connection issues](https://learn.microsoft.com/azure/azure-sql/database/troubleshoot-common-errors-issues), fetched 2026-08-29: errors 10928, 10929 and 10936. +- `sys.dm_db_resource_stats`, `sys.database_scoped_configurations` and `sys.dm_user_db_resource_governance` reference pages. +- Every container row in the availability table: measured directly against the local engine on 2026-08-29. diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/skill.spec.jsonc new file mode 100644 index 0000000..5fea3e4 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-resource-pressure/skill.spec.jsonc @@ -0,0 +1,171 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "diagnose-resource-pressure", + "domain": "performance-and-diagnostics", + "value": [ + "corrects-a-wrong-assumption", + "not-in-training-data" + ], + "correction": "Agents troubleshooting a slow or throttled Azure SQL Database reach for the readings that make sense against a single unshared engine, where the machine is the thing you measure. Azure SQL Database is a governed platform: every headline number is a percentage of a ceiling set by the service objective, and the view built to report those percentages, sys.dm_db_resource_stats, does not exist on the local Azure SQL Database container at all, raising Msg 208. The purchasing model is not guessable: dtu_limit is NULL on a vCore database and cpu_limit, the vCore count, is NULL on a DTU database, and on a DTU database the headline is not a column but Learn's formula avg_dtu_percent = MAX(avg_cpu_percent, avg_data_io_percent, avg_log_write_percent), with memory excluded because the data cache keeps avg_memory_usage_percent near 100 regardless of load. avg_cpu_percent and avg_instance_cpu_percent are percentages of two different limits and are not comparable with each other. Error 10928's 'request limit' wording names a worker limit rather than a connection count, kept only for backward compatibility, so Resource ID 1 is workers and Resource ID 2 is sessions, and MAXDOP multiplies workers per query far faster than session count does. MAXDOP itself is where a previous revision was wrong: it claimed sys.database_scoped_configurations reports 0 on Azure SQL Database while 8 applies invisibly underneath. Microsoft Learn contradicts that, setting the MAXDOP database-scoped configuration to 8 for each new single or elastic pool database since September 2020 and using that view as the way to determine it, so 8 in the cloud is the default, 0 in the cloud means the database predates the change, and 0 on the container is measured and is just the engine default. And the instance-level fallback is absent rather than restricted: sp_configure returns Msg 2812, could not find the stored procedure, not the option error Msg 15123 that would send a reader after a permission problem.", + "value_probe": { + "gate": "The answer measures the production database against the ceilings its service objective sets, refuses the laptop reading as evidence about the cloud database, and explains the number in the error message as something other than a count of open connections, before it says anything about changing the tier.", + "task": "Our production Azure SQL Database started timing out at peak yesterday and there are two things I cannot make sense of.\n\nFirst, I reproduced the workload against the Azure SQL Database container on my laptop, because I did not want to experiment in production. I tried:\n\n```sql\nSELECT TOP (20) * FROM sys.dm_db_resource_stats ORDER BY end_time DESC;\n```\n\nand it comes back as an invalid object name there, so I fell back to sys.dm_os_performance_counters, where the buffer cache hit ratio dips to about 94 percent under my load. Does that tell me production is short of memory and needs a bigger tier?\n\nSecond, the application log from the incident is full of 'The request limit for the database is 200 and has been reached'. Our connection pool is capped at forty connections and we run three application instances, so I cannot see how we would be anywhere near 200. This is a DTU based Standard database.\n\nWhat should I actually be measuring, and is scaling up the answer here?", + "states": [ + "Every headline number on this platform is a percentage of a ceiling set by the service objective rather than an absolute measurement of a machine.", + "The limit named in that error counts the workers a query consumes rather than the connections the application opens, so a small pool can still reach it.", + "A query running in parallel consumes several workers at once, which is how the count grows far faster than the connection count does.", + "The view that reports those percentages is a cloud only surface, so its absence on the local container is expected rather than a permission or context problem.", + "The DTU headline is derived from the largest of the processor, data input output and log write percentages, and the memory percentage is not part of it." + ], + "refutes": [ + "The 200 in that message is a limit on concurrent connections or sessions, so the pool must be opening more connections than the application thinks it is.", + "A buffer cache hit ratio measured against the local container is evidence about whether the production database is short of memory.", + "sys.dm_db_resource_stats is available on any Azure SQL Database engine including the local container, so its absence means a missing permission or the wrong database context.", + "A memory usage percentage near 100 on an Azure SQL Database means the database is out of memory and needs a larger tier.", + "sp_configure can be used on Azure SQL Database to inspect or change the instance level settings behind this." + ] + }, + "posture": [ + "read", + "inspect" + ], + "maturity": "preview", + "applies_to": [ + "azure-sql-db", + "azure-sql-db-container" + ], + "related": [ + "diagnose-connection-errors", + "diagnose-blocking-and-deadlocks", + "diagnose-slow-query", + "read-execution-plan", + "capture-with-extended-events", + "provision-azure-sql-db" + ], + "validation": { + "target": "container", + "setup": [ + "a live engine reachable over sqlcmd reporting SERVERPROPERTY('EngineEdition') = 5 and SERVERPROPERTY('Edition') = 'SQL Azure' (the local Azure SQL Database container)", + "a database created for the check, separate from other work on the same engine", + "target is container on purpose: every probe here was measured on the container, none has ever run against a real Azure SQL Database, and a probe carries no substrate of its own, so even the two Learn documents for both places are asserted only where they were measured" + ], + "assert": [ + "the purchasing model is read from dtu_limit and cpu_limit rather than assumed, and a DTU answer uses MAX(avg_cpu_percent, avg_data_io_percent, avg_log_write_percent) rather than looking for an avg_dtu_percent column", + "avg_cpu_percent and avg_instance_cpu_percent are not compared with each other as if they shared a scale, and avg_memory_usage_percent near 100 is not reported as a fault", + "a MAXDOP answer reads sys.database_scoped_configurations as the authority, states that 8 is the cloud default for databases created since September 2020, and does not claim a hidden value is applied beneath the reported one", + "a 10928, 10929 or 10936 diagnosis names Resource ID 1 as workers and Resource ID 2 as sessions, and is not reduced to 'lower the connection pool size' when MAXDOP is the actual multiplier", + "CPU-bound queries are surfaced by ordering sys.dm_exec_query_stats on total_worker_time directly, never by a worker-time-to-elapsed-time ratio filter that can silently invert" + ], + "probes": [ + { + "sql": "SELECT TOP 1 dtu_limit, cpu_limit FROM sys.dm_db_resource_stats ORDER BY end_time DESC;", + "expect": { + "error": 208 + }, + "why": "The headline claim, made executable. sys.dm_db_resource_stats is the only place dtu_limit and cpu_limit live, so on the container the question 'DTU or vCore, and how close to the limit' cannot be asked at all. Measured 2026-08-29: Msg 208, Invalid object name 'sys.dm_db_resource_stats'. A future build that answers instead of failing means the container gained the view, not that this probe was wrong." + }, + { + "sql": "SELECT TOP 1 * FROM sys.dm_user_db_resource_governance;", + "expect": { + "error": 208 + }, + "why": "Microsoft Learn names sys.dm_user_db_resource_governance as where the actual DTU and vCore resource limits are exposed, and as the view to compare when sizing a move between the two purchasing models. It is absent on the container, so a local run cannot see any governed limit. Measured 2026-08-29: Msg 208." + }, + { + "sql": "EXEC sp_configure 'max degree of parallelism';", + "expect": { + "error": 2812 + }, + "why": "The instance-level knob a SQL Server habit reaches for is not restricted here, it is absent. Learn lists sp_configure and RECONFIGURE among the syntax Azure SQL Database does not support, and Learn's sys.sp_configure page names SQL Server, Azure SQL Managed Instance and Analytics Platform System in its Applies to list and not Azure SQL Database. Measured 2026-09-03: Msg 2812, Could not find stored procedure 'sp_configure'. An earlier revision asserted Msg 15123, which would mean the procedure exists and refused the option, and sends a reader after a permission or option problem that is not there. ALTER DATABASE SCOPED CONFIGURATION is the surface." + }, + { + "sql": "SELECT [value] FROM sys.database_scoped_configurations WHERE [name] = 'MAXDOP';", + "expect": { + "rows": [ + [ + 0 + ] + ] + }, + "why": "The container reports 0, the plain engine default, because no Azure control plane set anything. A new Azure SQL Database reports 8, the service default since September 2020, and Learn treats this view as the authority rather than a value with something applied beneath it. Measured 2026-08-29. It is the half of that pair that can be run locally, and it stops the container's 0 being read as the cloud's answer." + }, + { + "sql": "SELECT COUNT(*) AS log_files FROM sys.dm_io_virtual_file_stats(DB_ID(), 2) WHERE io_stall_queued_write_ms >= 0;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The one IO diagnostic Learn documents for both places, including the io_stall_queued_read_ms and io_stall_queued_write_ms columns that separate delay added by IO governance from delay added by storage. One row for file_id 2, the log file of the scratch database, proves the function and the governance columns are present on the container. A missing column would fail as Msg 207 rather than pass quietly." + }, + { + "sql": "SELECT COUNT(*) AS governor_waits FROM sys.dm_os_wait_stats WHERE wait_type IN ('LOG_RATE_GOVERNOR', 'POOL_LOG_RATE_GOVERNOR', 'INSTANCE_LOG_RATE_GOVERNOR') AND waiting_tasks_count > 0;", + "expect": { + "rows": [ + [ + 0 + ] + ] + }, + "why": "Zero here is the point, not a null result. The container has no log rate governance to throttle against, so these three wait types never accumulate however hard the log is pushed, and a saturated container log shows as WRITELOG instead. The probe exists so that a clean local reading is never carried across as evidence that a cloud workload is not log-rate limited. Measured 2026-08-29: none present." + }, + { + "sql": "SELECT service_objective FROM sys.database_service_objectives WHERE database_id = DB_ID();", + "expect": { + "rows": [ + [ + "GP_Gen5_2" + ] + ] + }, + "why": "The container answers this view with a fixed service objective for every database, unrelated to the CPU or memory it was actually given, and nothing enforces it. Measured 2026-08-29: GP_Gen5_2. The probe pins the placeholder so that reading a tier, or any headroom, out of this view locally fails loudly rather than reading as a real provisioned size." + }, + { + "cmd": "sqlcmd '-?' 2>&1 | grep -q -- '-A,--dedicated-admin-connection'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The body tells a reader to reach for the diagnostic connection during a worker-limit incident, because ordinary connections are being refused at that moment. This proves the flag exists on the client a developer installs, with no connection and no credentials. Verified 2026-09-03 against the sqlcmd on this machine, which lists '-A,--dedicated-admin-connection' in its backwards-compatible flag help. It runs on the host because sqlcmd is a client tool, not something in the engine image." + } + , + { + "sql": "SELECT * FROM sys.dm_db_resource_stats;", + "expect": { + "error": 208 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. sys.dm_db_resource_stats is an Azure SQL Database view the container does not carry, so the first thing an answer reaches for is the first thing that fails locally. The skill's rule is not to read this as a broken database. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "EXEC sp_configure 'max degree of parallelism';", + "expect": { + "error": 2812 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. sp_configure is not there at all on the container, which is a different failure from the option error a reader would otherwise go hunting for. This is also the measurement that showed embeddings-and-external-models was attributing the wrong number to the same statement. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Diagnose resource pressure on my Azure SQL database, is it CPU, IO, memory, or the service tier?", + "My Azure SQL Database is hitting error 10928, what resource is actually exhausted?", + "Run a resource pressure investigation against this Azure SQL database and tell me what to do about it" + ], + "implicit": [ + "Queries that used to finish instantly are timing out under load, what changed on the database side?", + "Should I scale up my Azure SQL database or is a bad query the real problem?", + "sys.dm_os_performance_counters shows the buffer cache hit ratio dropping on my local container, does that mean I need a bigger service tier in the cloud?", + "the request limit for the database is 200 and has been reached, what does that mean" + ], + "negative": [ + "Why does my connection attempt fail with error 18456?", + "Show me the execution plan for this query and tell me why it chose a scan", + "Two sessions are stuck blocking each other, help me find the head blocker", + "Rewrite this query to use a better join order", + "Set up extended events to capture the text of long-running queries", + "Create a firewall rule so my laptop can reach the database" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query deleted file mode 120000 index a95d4af..0000000 --- a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query +++ /dev/null @@ -1 +0,0 @@ -../../../skills/diagnose-slow-query \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/SKILL.md b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/SKILL.md new file mode 100644 index 0000000..8aa4f26 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/SKILL.md @@ -0,0 +1,270 @@ +--- +name: diagnose-slow-query +description: >- + Triages a slow Azure SQL Database query into one of four causes before anyone touches an index + or a service tier: volatile, duration swings across executions; blocked, waiting on another + session; regressed, a worse plan replaced a good one; or growing, duration rises with data + volume. Reads Query Store runtime stats, plan history and per-plan waits, and knows where AUTO + capture mode silently drops the query asked about. Use when a query "used to be fast" or runs + inconsistently: "this took a second yesterday and ten today", "sometimes it's fast and + sometimes it isn't", "did last night's deployment make this slower". Ends in a diagnosis, not a + fix: CPU, memory, IO and tier go to diagnose-resource-pressure, blocking and deadlocks to + diagnose-blocking-and-deadlocks, the plan itself to read-execution-plan, a query Query Store + missed to capture-with-extended-events. +--- + +# Diagnose a slow Azure SQL Database query + +**This is a triage, not a fix.** It turns "this is slow" into one of four labels, volatile, blocked, +regressed, or growing, and hands the labeled query to the skill that answers that label. Answering +the label, reading the plan XML, or capturing an event session, belongs elsewhere. + +Measured on 2026-08-29 against a live engine reporting Edition `SQL Azure` and EngineEdition 5 +(`Microsoft SQL Azure (RTM) - 12.0.2000.8`). Column names, capture modes, duration units and the +plan forcing failure reasons were re-checked against Microsoft Learn on 2026-09-03. + +## The fact that changes how you look + +**Query Store is on by default and still does not hold a complete record of what ran.** A database +created moments before this was written already reported `actual_state_desc` `READ_WRITE`, so +enabling it is never the first step. Its capture mode there was `AUTO`, which ignores infrequent +queries and queries with insignificant compile and execution duration, against thresholds Microsoft +Learn describes only as internally determined. A query run four times and then flushed with +`sys.sp_query_store_flush_db` still produced zero rows in `sys.query_store_query`; setting +`QUERY_CAPTURE_MODE = ALL` captured the same shape on its next execution. **Absence from Query +Store is not evidence a query is fine.** Read the mode rather than assuming it: two Microsoft Learn +pages disagree about the default for SQL Server builds, `ALL` on one and `AUTO` on the other, so +`sys.database_query_store_options` is the only answer that is about your database. + +The plan cache is not a rescue for that gap. On this engine a query against a user table can carry +a cached compiled plan with a nonzero `usecounts` in `sys.dm_exec_cached_plans` and still return +zero matching rows in `sys.dm_exec_query_stats`, confirmed by an exact row count before and after +the query ran. + +A second trap sits next to the first. A query Query Store **did** capture is not stored as the +text that ran. Executed as +`SELECT COUNT(*) FROM dbo.q WHERE id>0 /*QSTIME_cheap4*/`, it is stored in `query_sql_text` as +`(@1 tinyint)SELECT COUNT(*) FROM [dbo].[q] WHERE [id]>@1`. Neither the literal nor the comment +appears anywhere, so a `LIKE` search for either returns the same empty, error free result the +`AUTO` gap produces, for a different reason. Re-measured on separate fresh databases on 2026-08-29: +comment stripping held on every capture, including a two table JOIN whose literal was **not** +rewritten, so only the comment stripping is unconditional. Treat the count that guarantees capture +as unknown: a cheap query stayed absent through four executions and a flush on two databases, while +a costlier JOIN run 30 times on a third was captured. + +## Step 1: is it running right now, and is it blocked + +Before reading any history, check the present: + +```sql +SELECT session_id, status, blocking_session_id, wait_type, total_elapsed_time, cpu_time, command +FROM sys.dm_exec_requests +WHERE session_id = ; +``` + +`blocking_session_id` other than 0 means this is not a Query Store investigation at all. Stop and +hand it to `diagnose-blocking-and-deadlocks`, which reads the chain and the head blocker. Plan +history on a session waiting for a lock answers the wrong question. + +## Step 2: find the query, and know when you will not + +```sql +SELECT qsq.query_id, qsqt.query_sql_text, qsp.plan_id +FROM sys.query_store_query qsq +JOIN sys.query_store_query_text qsqt ON qsq.query_text_id = qsqt.query_text_id +JOIN sys.query_store_plan qsp ON qsq.query_id = qsp.query_id +WHERE qsqt.query_sql_text LIKE '%%'; +``` + +**The fragment has to survive normalization, or this returns nothing for a query that is right +there.** Never search on a comment tag from the original statement; it is stripped on every +capture, whatever the query shape. A literal is a maybe rather than a no, and so are the brackets: +the `@N` placeholders and the bracketed identifiers arrive together, as the single parameterization +rewrite that the `(@1 tinyint)` prefix marks, so a statement the engine did not rewrite keeps its +literal and keeps `dbo.q` unbracketed. Search the bare object name, `LIKE '%q%'`, which matches +both shapes, then read `query_sql_text` back to see which one you got. Measured 2026-09-03: an +assignment `SELECT` was captured and found by its bare name in the same run in which its bracketed +form matched nothing. Matching on `object_id` only helps inside a module: Microsoft Learn states +it is populated only for a statement compiled from a Transact-SQL module, and is 0 for every ad +hoc batch, so it cannot tell two ad hoc queries apart. + +If nothing comes back after accounting for normalization, read the mode before concluding the query +never ran or was fast: + +```sql +SELECT actual_state_desc, query_capture_mode_desc, readonly_reason, + stale_query_threshold_days, max_storage_size_mb +FROM sys.database_query_store_options; +``` + +`AUTO` plus a one-off query is the likely explanation, not a clean bill of health: set +`QUERY_CAPTURE_MODE = ALL` or `CUSTOM` for the investigation, tell whoever owns the database +because `ALL` captures overhead too, and re-run the query. `READ_ONLY` +means Query Store stopped accepting data and `readonly_reason` says why: 65536 is the +`max_storage_size_mb` limit, 131072 the internal statement memory limit, 524288 the database out of +space. That is a capacity problem to raise, not a quiet workload. A query that must be caught on +its very next single occurrence is `capture-with-extended-events`'s job. + +**Reading these views needs `VIEW DATABASE STATE`.** Without it, `sys.dm_exec_query_stats` returns +`VIEW DATABASE PERFORMANCE STATE permission denied in database ''`, naming the newer +permission, but the older `VIEW DATABASE STATE` alone was sufficient here for that DMV and every +`sys.query_store_*` view tested, and Microsoft Learn agrees the greater permission satisfies it. +Check what the account already holds before asking for a grant. + +## Step 3: is it volatile + +A single `avg_duration` is not a verdict. Rank instability with the coefficient of variation, +`stdev_duration` divided by `avg_duration`, per plan per interval: + +```sql +SELECT rs.plan_id, rs.avg_duration, rs.stdev_duration, + rs.stdev_duration / NULLIF(rs.avg_duration, 0) AS coefficient_of_variation, + rs.count_executions +FROM sys.query_store_runtime_stats rs +WHERE rs.plan_id IN () + AND rs.execution_type = 0 +ORDER BY coefficient_of_variation DESC; +``` + +`execution_type = 0` keeps aborted and exception executions out of the ranking. Duration columns +are **microseconds**, so a one second execution reads as 1000000. + +Use this to **rank**, not to threshold. A coefficient of variation far above its neighbors, or +above its own earlier value, marks a query whose duration depends on something outside the query +text, usually the parameter values it was called with. That is volatile: the next question is which +parameter shape is expensive, not what to tune. + +## Step 4: is it regressed + +More than one `plan_id` for a `query_id` means the plan changed, and the query's own history says +whether that was for the better. In the interval still active one plan can hold several rows, one +flushed to disk and the rest in memory, so weight by `count_executions` instead of reading a row: + +```sql +SELECT qsp.plan_id, qsp.is_forced_plan, + SUM(rs.avg_duration * rs.count_executions) / SUM(rs.count_executions) AS weighted_duration, + SUM(rs.count_executions) AS executions +FROM sys.query_store_plan qsp +JOIN sys.query_store_runtime_stats rs ON qsp.plan_id = rs.plan_id +WHERE qsp.query_id = + AND rs.execution_type = 0 +GROUP BY qsp.plan_id, qsp.is_forced_plan +ORDER BY qsp.plan_id; +``` + +If an earlier plan performed better, **force it back before removing the worse one**: + +```sql +EXEC sys.sp_query_store_force_plan @query_id = , @plan_id = ; +``` + +This ran clean against the test engine, and so did `sys.sp_query_store_unforce_plan` afterward. +Forcing is reversible and immediate; deleting query or plan history is neither, and throws away the +evidence the next regression needs. Forcing does not guarantee the plan gets used, so confirm it +took rather than trusting the `EXEC`. + +## Step 5: is it growing + +A duration that climbs steadily rather than jumping once is a different problem: the plan has not +changed, the data underneath it has. Compare one `plan_id` across intervals: + +```sql +SELECT rsi.start_time, rs.avg_duration, rs.avg_logical_io_reads, rs.count_executions +FROM sys.query_store_runtime_stats rs +JOIN sys.query_store_runtime_stats_interval rsi + ON rs.runtime_stats_interval_id = rsi.runtime_stats_interval_id +WHERE rs.plan_id = +ORDER BY rsi.start_time; +``` + +Rising `avg_logical_io_reads` alongside rising `avg_duration`, same plan throughout, is a table or +index that outgrew the plan shape that suited it, an indexing or schema question, not a Query +Store one. `stale_query_threshold_days` bounds how far back this reaches; history that looks +unexpectedly short usually means that setting, not a gap in what ran. + +## Step 6: read the wait composition to route the next step + +```sql +SELECT ws.wait_category_desc, SUM(ws.total_query_wait_time_ms) AS total_wait_ms +FROM sys.query_store_wait_stats ws +WHERE ws.plan_id = +GROUP BY ws.wait_category_desc +ORDER BY total_wait_ms DESC; +``` + +The `_desc` column carries the category name, so no mapping from numeric `wait_category` is +needed. Route on the dominant category, do not diagnose it here: + +- `Lock`, `Latch` or `Buffer Latch`: contention, not a slow plan. Go to + `diagnose-blocking-and-deadlocks`. +- `CPU`, `Memory`, `Buffer IO` or `Tran Log IO`: a resource question. Go to + `diagnose-resource-pressure`. `Log Rate Governor` is a category too, but it reports a service + behaviour the container cannot show you, so treat it as cloud only evidence. +- No single category dominant and the plan looks wrong for the data: go to `read-execution-plan` + for the operator responsible. + +## Check it worked + +Two checks after acting on the verdict. Both are about the change, not Query Store. + +**One: a forced plan actually took.** Run the query once more, then read the plan rows back: + +```sql +SELECT plan_id, is_forced_plan, force_failure_count, + last_force_failure_reason, last_force_failure_reason_desc +FROM sys.query_store_plan +WHERE query_id = ; +``` + +Expect one row at `is_forced_plan` = 1 with `force_failure_count` = 0 and +`last_force_failure_reason` = 0, and every other plan for that query at `is_forced_plan` = 0. A +nonzero reason names the refusal: 8712 `NO_INDEX`, an index the plan needs is gone; 8698 `NO_PLAN`, +the plan could not be verified for this query; 8690 `HINT_CONFLICT`, a query hint contradicts it. +The count only moves on a recompile, not on every execution, which is why the re-run comes first. + +**Two: a new interval is measurably faster.** + +```sql +EXEC sys.sp_query_store_flush_db; +SELECT rsi.start_time, rs.plan_id, rs.count_executions, rs.avg_duration +FROM sys.query_store_runtime_stats rs +JOIN sys.query_store_runtime_stats_interval rsi + ON rs.runtime_stats_interval_id = rsi.runtime_stats_interval_id +WHERE rs.plan_id IN (, ) + AND rs.execution_type = 0 +ORDER BY rsi.start_time DESC; +``` + +Expect the newest `start_time` to carry the new `plan_id` only, its `count_executions` matching the +runs you just made, and its `avg_duration` below the old plan's. Durations are microseconds, so +two seconds down to 200 ms reads as roughly 2000000 against 200000, not 2 against 0.2. The old +`plan_id` still winning the newest interval means a forced plan did not take, and check one says why. +Unchanged durations on the new plan mean the plan was never the cause: the label was wrong, so go +back to step 6. + +## Do not + +- Do not report a query as fast, or as never having run, because it is missing from Query Store. + Read `query_capture_mode_desc` first. +- Do not treat `sys.dm_exec_query_stats` as a fallback source of truth when Query Store comes up + empty. A cached, used plan can still have zero rows there. +- Do not `LIKE` search `sys.query_store_query_text` for a comment tag, or for a bracketed + identifier, and conclude the query never ran. Match the bare object name instead. +- Do not read a verdict out of one `avg_duration`, or compare two plans without weighting by + `count_executions`. +- Do not call a forced plan done because the `EXEC` returned. Check `is_forced_plan` and + `last_force_failure_reason` after a re-run. +- Do not remove a query or a plan from Query Store when forcing the earlier plan already fixes it. +- Do not diagnose CPU, memory, IO or service tier pressure here, that is + `diagnose-resource-pressure`; do not build a blocking chain or read a deadlock graph, that is + `diagnose-blocking-and-deadlocks`; do not walk the plan XML operator by operator, that is + `read-execution-plan`. +- Do not leave `QUERY_CAPTURE_MODE` on `ALL` afterward. It captures every trivial query, at a cost + the database owner should choose. + +## References + +- [references/query-store-queries.md](references/query-store-queries.md): open it when the step 2 + search finds nothing and you need the before and after of a normalized capture, when you want + instability or multi plan queries ranked across the whole database, or when the permission error + blocks the read and you need the grant that cleared it. diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/references/query-store-queries.md b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/references/query-store-queries.md new file mode 100644 index 0000000..93531ff --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/references/query-store-queries.md @@ -0,0 +1,131 @@ +# Query Store queries for slow-query triage + +Every query in this file was run against a live engine reporting Edition `SQL Azure` and +EngineEdition 5 on 2026-08-29, against a database created for that purpose, not recalled from +memory. + +## Contents + +- [Rank instability across the whole database](#rank-instability-across-the-whole-database) +- [Find every query with more than one plan](#find-every-query-with-more-than-one-plan) +- [Query text is normalized, and comments do not survive](#query-text-is-normalized-and-comments-do-not-survive) +- [The permission error, verbatim](#the-permission-error-verbatim) + +## Rank instability across the whole database + +Widens step 3 of `SKILL.md` from one query to every query with more than one execution, so the +worst coefficient of variation in the database surfaces without knowing which query to ask about +first: + +```sql +SELECT TOP 20 qsq.query_id, qsp.plan_id, rs.avg_duration, rs.stdev_duration, + rs.stdev_duration / NULLIF(rs.avg_duration, 0) AS coefficient_of_variation, + rs.count_executions +FROM sys.query_store_query qsq +JOIN sys.query_store_plan qsp ON qsq.query_id = qsp.query_id +JOIN sys.query_store_runtime_stats rs ON qsp.plan_id = rs.plan_id +WHERE rs.count_executions > 1 +ORDER BY coefficient_of_variation DESC; +``` + +Run against the test database this returned seven queries with more than one execution, headed by +a coefficient of variation of 0.62 on a query executed three times with widely different row +counts, and tailed by one at 0.065 that is effectively constant. The ranking, not either number +alone, is what points at the one worth reading further. + +`count_executions > 1` matters: a coefficient of variation from two executions is barely a sample, +and a `query_id` with `count_executions = 1` divides by an `avg_duration` that is also its only +data point, which ranks as perfectly stable for having nothing to vary against. + +## Find every query with more than one plan + +Step 4 needs a `query_id` to inspect. This finds every one with plan history worth reading, across +the whole database, in one pass: + +```sql +SELECT query_id, COUNT(*) AS plan_count +FROM sys.query_store_plan +GROUP BY query_id +HAVING COUNT(*) > 1 +ORDER BY plan_count DESC; +``` + +On the test database, one `query_id` carried two plans after a plan was deliberately forced and +then unforced during verification, which is the same shape a real plan change leaves behind: +`is_forced_plan` on `sys.query_store_plan` distinguishes a plan the optimizer chose from one an +operator pinned, and is worth selecting alongside `plan_id` when reading the result. + +## Query text is normalized, and comments do not survive + +Even a query Query Store did capture is not stored as the text that ran. Executed as: + +```sql +SELECT COUNT(*) FROM dbo.q WHERE id>0 /*QSTIME_cheap4*/ +``` + +it is stored in `sys.query_store_query_text.query_sql_text` as: + +``` +(@1 tinyint)SELECT COUNT(*) FROM [dbo].[q] WHERE [id]>@1 +``` + +Checked directly on the test engine: a `LIKE '%QSTIME_cheap4%'` and a `LIKE '%id>0%'` both return +zero rows against this exact query, captured moments earlier under `QUERY_CAPTURE_MODE = ALL`. The +literal is gone, replaced by a typed parameter placeholder, `@1 tinyint`. The comment is gone +entirely. Identifiers gained brackets they did not have in the original text. + +**The brackets and the `@N` placeholder are the same event, and it does not happen to every +query.** The `(@1 tinyint)` prefix is what Microsoft Learn calls the marker of a statement the +engine parameterized: the names and types of the parameters come before the text of the submitted +batch. The bracketing rides along with that rewrite, so a statement the engine did not rewrite +keeps its identifiers as the author wrote them. Learn also warns that parameter names, which +literals get parameterized, and the spacing can change between builds, a second reason not to match +on the rewritten shape. + +Measured against that on 2026-09-03, in one run against one database: `SELECT @rc = COUNT(*) FROM +dbo.probe_qs_norm WHERE id > 0` was captured, was found by a search for `probe_qs_norm`, and was +NOT found by a search for `[dbo].[probe_qs_norm]`. The bracketed pattern that matched the +single-table case above missed this one, in the same view, for the same table. + +So search the bare object name, which matches a rewritten capture and an unrewritten one alike, +then read the text back rather than guessing which you got: + +```sql +SELECT query_sql_text +FROM sys.query_store_query_text +WHERE query_sql_text LIKE '%q%'; +``` + +If you do search a bracketed pattern, `[[]` escapes a literal `[` for `LIKE`, since `[` is itself a +wildcard character in T-SQL pattern matching, and `LIKE '%[[]dbo].[[]q]%'` is what matched the +rewritten row above. + +Both this and the `AUTO` gap above were re-measured on separate, freshly created databases on +2026-08-29: the single-table case repeated exactly, literal stripped, comment stripped, brackets +added, while a two table JOIN was captured with its comment stripped and its literal left in place. +**Comment stripping was the one constant across every shape; the literal rewrite and the brackets +were not.** + +## The permission error, verbatim + +A login or user with `CONNECT` only, and neither `VIEW DATABASE STATE` nor +`VIEW DATABASE PERFORMANCE STATE`, gets this from `sys.dm_exec_query_stats`: + +``` +Msg 262, Level 14, State 1, Server SQL Azure, Line 1 +VIEW DATABASE PERFORMANCE STATE permission denied in database ''. +``` + +Granting `VIEW DATABASE STATE` alone, the older and more commonly already-granted permission, was +sufficient to clear this for both `sys.dm_exec_query_stats` and every `sys.query_store_*` catalog +view tested: `sys.query_store_query`, `sys.query_store_plan`, `sys.query_store_runtime_stats`, and +`sys.query_store_wait_stats`. The error names the newer permission; do not assume that name is the +only one that satisfies it before checking what the account already has with: + +```sql +SELECT permission_name, state_desc +FROM sys.database_permissions dp +JOIN sys.database_principals p ON dp.grantee_principal_id = p.principal_id +WHERE p.name = ''; +``` + diff --git a/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/skill.spec.jsonc new file mode 100644 index 0000000..6655fef --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/diagnose-slow-query/skill.spec.jsonc @@ -0,0 +1,134 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "diagnose-slow-query", + "domain": "performance-and-diagnostics", + "value": ["not-in-training-data", "corrects-a-wrong-assumption"], + "correction": "Agents troubleshooting a slow Azure SQL Database query treat Query Store as an automatic, complete record of everything that ran, because it is enabled by default on every database, when its default AUTO capture mode applies internal thresholds that silently drop ad hoc and infrequently executed queries, so the exact query someone ran once and is asking about can be completely absent from every Query Store view even immediately after a manual flush, and checking the plan cache instead does not rescue the diagnosis, because a query can hold a cached compiled plan with a nonzero use count and still have zero matching rows in sys.dm_exec_query_stats.", + "value_probe": { + "gate": "The answer treats the absence of the statement from the built-in history as a property of what that history captures rather than as evidence about the query, and gives the reader a way to make this statement recordable or to reach the evidence another way.", + "task": "One of our reports on Azure SQL Database took about forty seconds this morning when it normally takes two. It has already finished, so I cannot catch it while it is running. I went looking in the database's own built-in history of what ran and I cannot find the statement at all, not even straight after re-running it once by hand to see if that helped. Does that mean it was never slow, or that something is off about how I am looking? Walk me through how you would work out what happened and which numbers you would want in front of you.", + "states": [ + "The default capture mode applies internal thresholds that drop ad hoc and infrequently executed statements.", + "Duration columns in that history are in microseconds, so a one second execution reads as one million.", + "A statement can hold a cached plan with a nonzero use count and still have no matching rows in the execution statistics view.", + "A single average duration is not a verdict, because the spread across executions is what separates the possible causes." + ], + "refutes": [ + "Everything that runs is recorded, so a statement missing from the history did not run slowly or did not run at all.", + "Flushing the history to disk makes an ad hoc statement that was dropped appear in it.", + "Looking in the plan cache instead reliably rescues the diagnosis when the statement is missing from the history.", + "The duration columns in that history are in milliseconds." + ] + }, + "posture": ["read", "inspect", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "diagnose-resource-pressure", + "diagnose-blocking-and-deadlocks", + "read-execution-plan", + "capture-with-extended-events" + ], + "validation": { + "target": "both", + "setup": [ + "a table with enough rows that a scan and a seek cost visibly different amounts", + "the same query executed a handful of times with parameter values that make its row count vary widely", + "a second, different plan produced for the same query id, for example by forcing a table scan once and an index seek once", + "a query executed exactly once under the default AUTO query capture mode, never repeated", + "a session left open with an uncommitted transaction while a second session runs the query under test" + ], + "assert": [ + "sys.database_query_store_options reports actual_state READ_WRITE on a newly created database with no configuration step taken", + "a query executed only once under AUTO capture mode is correctly named as possibly absent from sys.query_store_query, rather than assumed present", + "the coefficient of variation, stdev_duration divided by avg_duration from sys.query_store_runtime_stats, is used to rank instability rather than a single avg_duration value read alone", + "a query id carrying more than one plan_id in sys.query_store_plan is read as a plan change over time, and the better plan is forced with sys.sp_query_store_force_plan rather than the worse plan simply being removed", + "a session with a nonzero blocking_session_id in sys.dm_exec_requests is diagnosed as blocked and handed to diagnose-blocking-and-deadlocks instead of being treated as a plan or resource problem", + "wait composition read from sys.query_store_wait_stats is used to decide whether the next step is diagnose-resource-pressure or diagnose-blocking-and-deadlocks, not guessed" + ], + "probes": [ + { + "sql": "SELECT actual_state_desc, CASE WHEN query_capture_mode_desc IN ('ALL', 'AUTO', 'NONE', 'CUSTOM') THEN 1 ELSE 0 END AS capture_mode_is_a_documented_value FROM sys.database_query_store_options;", + "expect": { + "rows": [ + [ + "READ_WRITE", + 1 + ] + ] + }, + "why": "The claim the first paragraph turns on: Query Store is already collecting on a database nobody configured, so enabling it is never step one. Measured 2026-08-29 on a database created moments earlier, and Microsoft Learn states Query Store is enabled by default for new Azure SQL Database databases and cannot be turned off there. The capture mode is asserted only as one of the four documented values rather than as AUTO, because two Learn pages disagree about the default for SQL Server builds and the skill tells the reader to read the mode rather than assume it. No probe asserts that AUTO drops a given query: the skill's own measurements show a cheap query absent after four executions while a costlier JOIN run 30 times was captured, so the threshold is not a fixed count and an absence assertion would be flaky. That claim stays prose." + }, + { + "sql": "SET NOCOUNT ON;\nALTER DATABASE CURRENT SET QUERY_STORE (QUERY_CAPTURE_MODE = ALL);\nCREATE TABLE dbo.probe_qs_norm (id int NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_qs_norm VALUES (1), (2);\nDECLARE @rc int, @tag nvarchar(40) = N'PROBEQS' + N'TAG', @obj nvarchar(40) = N'probe_qs' + N'_norm';\nSELECT @rc = COUNT(*) FROM dbo.probe_qs_norm WHERE id > 0 /*PROBEQSTAG*/;\nEXEC sys.sp_query_store_flush_db;\nSELECT (SELECT COUNT(*) FROM sys.query_store_query_text WHERE query_sql_text LIKE N'%' + @tag + N'%') AS comment_matches, CASE WHEN EXISTS (SELECT 1 FROM sys.query_store_query_text WHERE query_sql_text LIKE N'%COUNT(*)%' AND query_sql_text LIKE N'%' + @obj + N'%') THEN 1 ELSE 0 END AS found_by_bare_name, CASE WHEN EXISTS (SELECT 1 FROM sys.query_store_query_text WHERE query_sql_text LIKE N'%COUNT(*)%' AND query_sql_text LIKE N'%[[]dbo].[[]probe_qs_norm]%') THEN 1 ELSE 0 END AS found_by_bracketed_name;", + "expect": { + "rows": [ + [ + 0, + 1, + 0 + ] + ] + }, + "why": "The normalization trap, made executable, in the three columns that tell its halves apart. The comment never survives capture, so column one is 0. The statement IS captured and is found by the bare object name, so column two is 1. The bracketed form is NOT there, so column three is 0, and that is the correction this probe exists for: the brackets and the @N placeholders arrive together as one parameterization rewrite, marked by the (@1 tinyint) prefix Microsoft Learn documents, so they are absent from a statement the engine did not rewrite. Measured by the lane on 2026-09-03, when an earlier version of this probe searched only the bracketed form and returned 0 while the force plan probe found the same statement by its bare name in the same run: captured, and unbracketed. Both patterns also require COUNT(*), which the fixture INSERT does not have, so neither column can be satisfied by anything but the workload statement; the tag and the object name are rebuilt from two literals each so the searching statement cannot match itself. QUERY_CAPTURE_MODE is set to ALL first, because the default AUTO is what the skill says drops a query this cheap." + }, + { + "sql": "CREATE TABLE dbo.probe_qs_force (id int NOT NULL PRIMARY KEY, pad char(50) NOT NULL);\nINSERT INTO dbo.probe_qs_force (id, pad) SELECT TOP (200) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)), 'x' FROM sys.all_objects;\nALTER DATABASE CURRENT SET QUERY_STORE (QUERY_CAPTURE_MODE = ALL);\nDECLARE @rc int, @qid bigint, @pid bigint;\nSELECT @rc = COUNT(*) FROM dbo.probe_qs_force WHERE id > 10;\nEXEC sys.sp_query_store_flush_db;\nSELECT TOP (1) @qid = q.query_id, @pid = p.plan_id FROM sys.query_store_query q JOIN sys.query_store_query_text t ON q.query_text_id = t.query_text_id JOIN sys.query_store_plan p ON p.query_id = q.query_id WHERE t.query_sql_text LIKE '%COUNT(*)%probe%force%' AND t.query_sql_text NOT LIKE '%query_store%' ORDER BY q.query_id DESC;\nEXEC sys.sp_query_store_force_plan @query_id = @qid, @plan_id = @pid;\nSELECT is_forced_plan, force_failure_count, last_force_failure_reason FROM sys.query_store_plan WHERE plan_id = @pid;", + "expect": { + "rows": [ + [ + 1, + 0, + 0 + ] + ] + }, + "why": "Step 4 tells the reader to force the better plan, and the Check it worked section tells them to read the result back rather than trust the EXEC. This runs both halves: sys.sp_query_store_force_plan with the two parameters Microsoft Learn documents, then the three columns that say whether the forcing took. is_forced_plan 1 with force_failure_count 0 and last_force_failure_reason 0 is the clean outcome; Learn lists 8712 NO_INDEX, 8698 NO_PLAN and 8690 HINT_CONFLICT among the nonzero reasons. Forcing requires ALTER on the database, which the administrative connection a probe runs on has. The workload query assigns its count to a variable so the batch emits one rowset; the recorded text still carries COUNT(*) and the table name the lookup matches on, with no brackets assumed. This probe passing on 2026-09-03 is what proves an assignment SELECT is captured at all, which is the half of the normalization probe that the bracketed search could not distinguish. The lookup excludes text mentioning query_store, because under QUERY_CAPTURE_MODE = ALL the lookup statement is itself captured and its own pattern literal would otherwise match it, which would force a plan on the wrong query and still return a passing row." + }, + { + "sql": "SELECT ws.wait_category_desc, SUM(ws.total_query_wait_time_ms) AS total_wait_ms FROM sys.query_store_wait_stats ws WHERE ws.plan_id = -1 GROUP BY ws.wait_category_desc ORDER BY total_wait_ms DESC;", + "expect": { + "rows": [] + }, + "why": "Step 6 routes on wait category, so the view and both of the columns it names have to exist on the engine this skill runs against. sys.query_store_wait_stats is documented for SQL Server 2017 and later and for Azure SQL Database, and carries wait_category_desc alongside the numeric wait_category, which is why no mapping table is needed in the body. plan_id = -1 can match nothing, so an empty result is the whole claim: the statement parsed and every column resolved." + }, + { + "sql": "SELECT CASE WHEN EXISTS (SELECT 1 FROM sys.dm_exec_requests WHERE session_id = @@SPID AND blocking_session_id = 0) THEN 1 ELSE 0 END AS own_request_is_unblocked;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "Step 1 sends the reader to sys.dm_exec_requests before any Query Store read, and routes on blocking_session_id. This asserts the DMV and that column exist and that an unblocked request reads 0 there, which is the value the whole step-1 branch is written against. The probe's own session is by definition not blocked while it is executing." + } + ] + }, + "triggering": { + "explicit": [ + "This query used to run in under a second and now it takes ten seconds, find out why", + "Diagnose why this stored procedure is slow", + "Use diagnose-slow-query on this report query" + ], + "implicit": [ + "Sometimes this dashboard finishes in two seconds and sometimes it takes forty, same parameters", + "This query got slow right after last night's deployment and nothing else changed", + "Query Store shows nothing for the query I just ran, does that mean it was fast", + "The nightly job used to take ten minutes and now it takes forty five, and it keeps creeping up", + "the same query used to be instant and now it reads the whole table, nothing changed" + ], + "routing": [ + "why is this slow", + "this report takes forever now" + ], + "negative": [ + "Why is CPU pinned at 100 percent on this database", + "Two sessions look stuck waiting on each other, one of them is a deadlock victim", + "Read this execution plan XML and tell me which operator is expensive", + "Set up a session to capture the next occurrence of this automatically", + "Rewrite this query so it does not use a cursor", + "How do I create an index on this table" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth b/plugins/microsoft-sql-ssms/skills/entra-id-auth deleted file mode 120000 index e45999d..0000000 --- a/plugins/microsoft-sql-ssms/skills/entra-id-auth +++ /dev/null @@ -1 +0,0 @@ -../../../skills/entra-id-auth \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth/SKILL.md b/plugins/microsoft-sql-ssms/skills/entra-id-auth/SKILL.md new file mode 100644 index 0000000..e633a8a --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth/SKILL.md @@ -0,0 +1,382 @@ +--- +name: entra-id-auth +description: >- + Takes an application identity to a passwordless connection to Azure SQL Database, and diagnoses + it when that fails: sets the Microsoft Entra administrator, creates the database user for a + managed identity or service principal, and grants roles. Use for "set up Microsoft Entra + authentication for Azure SQL", "connect with a managed identity", "stop putting the database + password in configuration", or "turn on Microsoft Entra-only authentication", and for the + container once its MSSQL_AAD_ variables are set. Owns failures after a credential was + evaluated: Msg 33134 "Principal could not be resolved", Msg 33131 "duplicate display name", + 18456 "Login failed for user", 4060 "Cannot open database". What fails before that is + diagnose-connection-errors; the connection code itself is connect-from-dotnet, + connect-from-python or connect-from-typescript-and-node. +--- + +# Microsoft Entra ID authentication for an application identity + +Passwordless connection is an ordered procedure whose failures do not arrive in order: steps succeed +while achieving nothing, and the errors name the wrong component. + +Every flag below was read from the tool on 2026-09-03: Azure CLI 2.90.0, `sqlcmd` 1.10.0, `azd` +1.32.0, and the container behaviour was measured the same day. Cloud engine behaviour and message +text are from Microsoft Learn, cited in `references/`. + +## What this skill owns, and what it does not + +**Owns** the server-side identity configuration, the database user, the grant, the authentication +keyword, and every failure about **who the caller is**. Route the rest. + +| Question | Skill | +|---|---| +| Retry, encryption, pooling, and whether the container has Entra ID configured at all | `connect-to-azure-sql` | +| Drivers and per-language syntax | `connect-from-dotnet`, `connect-from-python`, `connect-from-typescript-and-node` | +| The server, the database, the firewall rule | `provision-azure-sql-db` | +| Transport, pre-login, certificates, timeouts, the `40xxx` gateway family | `diagnose-connection-errors` | +| Which roles the application should hold | `least-privilege-database-roles` | +| Server-level Entra logins and fixed server roles | `entra-logins-and-server-roles` | +| Identity through a hosting service, and the deployment | `managed-identity-across-azure-services`, `deploy-app-to-azure` | + +**The split, stated once.** `diagnose-connection-errors` owns failures before a credential is +evaluated; this skill owns those after it: `18456`, `4060`, `33134`, `33131`, `37545`. A failure +carrying no SQL error number is neither. + +## 18456 and 4060 arrive after the credential + +Both read as "login failed", neither is a network problem, and no edit to the server name, +encryption or timeout fixes either one. + +- **`18456`** means the credential was evaluated and refused. Naming + `` it means the opposite of what it looks like: the token was + **accepted**, and no matching principal exists in the database. +- **`4060`** means the login is valid and has no user in **that** database. After a deployment that + is almost always a missing database user: a login lives on the server, a user in the database, and + deployment tooling creates neither. + +Both are answered by step 5, run on the user database. `40532` reads the same and is the gateway +refusing before a database was reached, so it is `diagnose-connection-errors`. + +## The order that works + +Steps 2, 5 and 7 are the ones that fail quietly. + +1. **Set the Microsoft Entra administrator on the logical server.** Nothing works until it exists, + and the CLI takes an object id rather than a name, so look it up first. + + ```bash + az ad user list --display-name "" --query "[].{id:id, upn:userPrincipalName}" -o table + az sql server ad-admin create -g -s \ + --display-name "" --object-id + ``` + +2. **Decide whether the logical server needs its own identity.** It does **only** if a service + principal, not a signed-in person, runs `CREATE USER`, the normal case for a pipeline. See + [Msg 33134](#msg-33134-does-not-mean-what-it-says). + +3. **Give the application an identity, and take its client id from the deployment, not the portal.** + `azd` prints the client id among the environment values only if the infrastructure declared it + as an output, so add that output rather than copying a GUID. + + ```bash + azd env get-values -e + az ad sp show --id --query "{appId:appId, objectId:id}" + ``` + + `appId` is the client id step 5 wants. `objectId` is the enterprise application object id, which + only `WITH OBJECT_ID` wants. They are not interchangeable. + +4. **Connect to the user database with a Microsoft Entra authenticated connection.** Not `master`, + and not a SQL login. + + ```bash + sqlcmd -S .database.windows.net -d -U -G -l 30 + ``` + +5. **Create the database user.** `CREATE USER` adds the user to the **current** database, so the + database in step 4 is the one that gets it. Run against `master` it is valid, succeeds, and + reports nothing; the application then fails against the user database with correct code. + + ```sql + CREATE USER [] FROM EXTERNAL PROVIDER; + ``` + +6. **Grant roles, and name them.** Add `db_ddladmin` only if the application applies schema. A grant + made after the application first connected does not take until the host's token cache turns over, + so the grant is correct and the application keeps failing. + + ```sql + ALTER ROLE db_datareader ADD MEMBER []; + ALTER ROLE db_datawriter ADD MEMBER []; + ``` + +7. **Write the connection string**, from + [the per-driver table](#the-authentication-keyword-is-not-the-same-string-twice). + +8. **Check it worked from the database**, below. + +## Msg 33134 does not mean what it says + +```output +Msg 33134, Level 16, State 1, Line 1 +Principal 'test-user' could not be resolved. +Error message: 'Server identity is not configured. ...' +``` + +The first line sends an agent to check the principal name, which is almost always fine. **The fault +is on the logical server.** A **Microsoft Entra user** running `CREATE USER` is impersonated and +Graph queried with that user's permissions; an application cannot impersonate another application, +so a **service principal** falls back to the **server identity**. With no server identity, or one +that cannot read Graph, the lookup fails and the message blames the principal, which is why the same +statement works by hand and fails in a pipeline. + +```bash +az sql server update -g -n -i +az sql server show -g -n --query identity +``` + +**The flag is `-i`, long form `--assign_identity` with an underscore.** Measured 2026-09-03 on Azure +CLI 2.90.0: the hyphenated `--assign-identity` exits with `unrecognized arguments`, so a script +carrying it assigns nothing and the next `CREATE USER` still raises `33134`. + +Then, in order of tenant privilege needed: grant that identity Microsoft Graph read, either three +application permissions or the broader `Directory Readers` role, both needing a `Privileged Role +Administrator`; or put it in a role-assignable group holding `Directory Readers`, the production +shape, because a group owner can then add servers; or skip the lookup with the third `CREATE USER` +form below, the answer wherever that privilege is absent. Open +[references/identity-errors.md](references/identity-errors.md) before attempting any of the three, +because it names the exact Graph permissions. + +## Three forms of CREATE USER, and when each is forced + +| Form | Looks the principal up | Use it when | +|---|---|---| +| `CREATE USER [] FROM EXTERNAL PROVIDER;` | Yes | The default. A person is running it, or the server identity is configured | +| `CREATE USER [] FROM EXTERNAL PROVIDER WITH OBJECT_ID = '';` | Yes | The display name is **not unique** in the tenant, so the plain form fails | +| `CREATE USER [] WITH SID = , TYPE = E;` | **No** | No Graph permission is available, or Msg 33134 has to be worked around. Needs an engine that has Microsoft Entra configured, so on a bare container it raises `40530` | + +Agents paraphrase the third form into something that does not run, because the id must be converted +to binary. Open [references/identity-errors.md](references/identity-errors.md) and copy the worked +block there: `TYPE = E` covers users, applications and managed identities, `TYPE = X` groups. + +**Nothing validates that id**, so a wrong value creates a user no token will ever match and it reads +back as a healthy row. For a **user or group** it is the **object id**; for a **service principal or +managed identity** the **application (client) id**. + +**`WITH OBJECT_ID` is not the general fix and does not answer Msg 33134.** It exists for one problem: +Microsoft Entra ID permits duplicate application display names and the engine requires a unique one. +That produces a different error. + +```output +Msg 33131, Level 16, State 1, Line 4 +Principal 'myapp' has a duplicate display name. +``` + +Three traps. **`WITH OBJECT_ID` is a modifier on `FROM EXTERNAL PROVIDER`, not an alternative to +it**: on its own the statement is `Msg 37546, Can only specify object_id when creating user from +external provider`, measured 2026-09-08. The object id must exist in this tenant or the statement +fails with `Msg 37545`. And the **Object ID on an app registration is not the one on its service +principal**: this clause wants the enterprise application one, `objectId` in step 3. + +## A SQL-authenticated administrator cannot do any of this + +Learn is explicit: only Microsoft Entra users can create other Entra users in Azure SQL Database, +and no SQL-authenticated user can, the server administrator included, which is why "I am the admin +and it still fails" is a dead end. The permission needed is `ALTER ANY USER`, carried by `db_owner`, +but no grant substitutes for the connection being Entra authenticated. The `WITH SID` form is the +exception: it performs no external lookup. + +## The name to put in the brackets + +A wrong name produces **`Msg 33130`**, `Principal '' could not be found or this principal type +is not supported`, and reads as a permissions problem. Measured on two different logical servers, on +2026-09-05 and again on 2026-09-08: an unknown address, a bare guid and a non-existent application +name all answered 33130. `Msg 33134` is a different path, above; `Msg 33131` is the duplicate +display name, confirmed 2026-09-08 by `CREATE USER [Microsoft Graph] FROM EXTERNAL PROVIDER`. The one +nobody guesses: a **system-assigned** identity on a deployment slot is `/slots/`, +which is why an application works in production and fails in staging. Open +[references/identity-errors.md](references/identity-errors.md) before typing a name you inferred, +because it tables all five principal kinds. + +## The authentication keyword is not the same string twice + +Every value below is plausible in every driver and each is valid in only one of them. + +| Stack | Passwordless default | Managed identity, explicit | User-assigned client id goes in | +|---|---|---|---| +| .NET, `Microsoft.Data.SqlClient` | `Authentication=Active Directory Default` | `Active Directory Managed Identity`, alias `Active Directory MSI` | `User Id=` | +| Python, `mssql-python` | `Authentication=ActiveDirectoryDefault` | `Authentication=ActiveDirectoryMSI` | `UID=` | +| Python, `pyodbc` on ODBC Driver 18 | **no such value exists** | `Authentication=ActiveDirectoryMsi` | `UID=`, object id on most hosts | +| Node.js, `mssql` on `tedious` | `type: 'azure-active-directory-default'` | `azure-active-directory-msi-app-service`, `azure-active-directory-msi-vm` | `authentication.options.clientId` | +| Java, the Microsoft JDBC driver | `authentication=ActiveDirectoryDefault` | `ActiveDirectoryManagedIdentity`, alias `ActiveDirectoryMSI` | `msiClientId`, or `user` | + +**In production, name the managed identity mode, not the default.** The default walks a chain of +credential providers, which Microsoft warns costs response time, and can pick up a developer +credential on a machine that has one. + +Two facts a plausible guess gets wrong. **ODBC has no `ActiveDirectoryDefault`** and spells managed +identity `ActiveDirectoryMsi`, mixed case, so Python is two stories: `mssql-python` has a default +mode and uses `ActiveDirectoryMSI`, `pyodbc` has neither. **In Node.js the client id is nested**, in +`authentication.options.clientId`, not the connection `options` bag beside `encrypt`. + +Open [references/driver-auth-keywords.md](references/driver-auth-keywords.md) before writing one of +these values, because each list there is the driver's complete accepted set and a value outside it +is not a value. A driver that names its own set, such as `sqlcmd`, is faster to ask than to recall. + +**`Microsoft.Data.SqlClient` 7.0 needs the `Microsoft.Data.SqlClient.Extensions.Azure` package**, or +every `Active Directory *` mode fails at run time; connection strings do not change. `Active +Directory Password` is deprecated across the drivers and `[Obsolete]` in 7.0, because it rests on +the resource owner password credentials grant. + +```bash +dotnet add package Microsoft.Data.SqlClient.Extensions.Azure +``` + +## System-assigned or user-assigned + +Two decisions, confused because both use the same words. **The server identity** reads Microsoft +Graph and matters only for step 2; a user-assigned one survives the server and holds the Graph +permissions once for many. **The application identity** is what the application authenticates as, +and changes the connection string **only if user-assigned**, which must name its client id because +the host may carry several. + +## On the Azure SQL Database container, the shape is login then user + +The container reads a certificate, the application (client) id and the tenant id from three +environment variables set at creation. One command says which case you are in. + +```bash +docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' | grep -c '^MSSQL_AAD_' +``` + +`0` means Entra ID was never configured: an Entra login returns `18456`, `Login failed for user ''`, +which is `connect-to-azure-sql`, not this skill, and the error below. `3` means `MSSQL_AAD_CLIENT_ID`, +`MSSQL_AAD_PRIMARY_TENANT` and `MSSQL_AAD_CERTIFICATE_FILE_PATH` are set and the database side is +yours. Learn documents that trio, and the statements below, for the SQL Server on Linux image and not +for this one, so confirm the engine agreed rather than assuming: + +```bash +docker exec bash -c "grep Entra /var/opt/mssql/log/errorlog" +``` + +Expected: `Microsoft Entra ID authentication is enabled.` The principal is then a server login and +a database user from that login, not a contained user: + +```sql +CREATE LOGIN [] FROM EXTERNAL PROVIDER; +CREATE USER [] FROM LOGIN []; +``` + +### Msg 40530 blames the batch, and the batch is fine + +In the `0` case the no-lookup form is refused too, by a message naming the wrong thing. Measured +2026-09-03 with no `MSSQL_AAD_` variables: + +```output +Msg 40530, Level 16, State 1, Line 1 +The CREATE USER statement must be the only statement in the batch. +``` + +It **was** the only statement in the batch. `TYPE = X` fails identically, and so does every rewrite +tried, `EXEC (@cmd)` and `GO` separation included, so a reader who trusts the text rewrites the batch +forever and never succeeds. `CREATE USER [] WITHOUT LOGIN;`, the same batch shape, succeeds, +which rules the batch out. **The fix is server configuration, not SQL**: the engine has no Microsoft +Entra configuration and refuses the external principal types. Set the three variables and recreate +the container, or run against Azure SQL Database. Open +[references/identity-errors.md](references/identity-errors.md) before believing the number, because +Microsoft Learn documents `40530` only as the batch rule and that disagreement is recorded there. + +Two things have no local equivalent, so a green local run proves nothing about either: the **server +identity** and its Graph permission, which `Msg 33134` is about, and **Entra-only authentication** +with its Azure Policy enforcement. + +## Microsoft Entra-only authentication + +Enabling it disables SQL authentication at the **server** level, for every database and the server +administrator too. Existing SQL logins are not removed and new ones can still be created; they +simply cannot connect. + +```bash +az sql server ad-only-auth enable -g -n +az sql server ad-only-auth get -g -n +``` + +**The administrator must be set first**, or enabling fails through every interface, and it cannot be +removed until the setting is disabled again. Enabling also silently disables elastic jobs, SQL Data +Sync, SQL Insights, `EXEC AS` for Entra group members and some change data capture paths. + +**Enforcing it takes two Azure Policy definitions, not one**: one evaluates a server at creation, +the other the setting on an existing server. Open +[references/entra-only-and-policy.md](references/entra-only-and-policy.md) before enabling this on a +shared server or naming a policy in a plan, because it carries the full disabled-feature list, the +role split, and the two definition ids. + +## Check it worked + +Run the first two against the **user database**, never `master`. + +**1. The principal exists and its stored id is the identity you meant.** + +```sql +SELECT name, type, type_desc, + CAST(CAST(sid AS varbinary(16)) AS uniqueidentifier) AS entra_id, + DATALENGTH(sid) AS sid_bytes +FROM sys.database_principals +WHERE type IN ('E', 'X'); +``` + +Expected: `type_desc` is `EXTERNAL_USER` for an application or a person, `EXTERNAL_GROUP` for a +group. `entra_id` must equal the client id the workload runs as; a row carrying the wrong one looks +exactly like success. `sid_bytes` is `16` for a contained user, `18` for one from a server login. On +a container with no `MSSQL_AAD_` variables it returns nothing: no `E` or `X` principal exists there. + +**2. The roles are the ones you named, and nothing wider.** + +```sql +SELECT r.name AS role_name, m.name AS member_name +FROM sys.database_role_members rm +JOIN sys.database_principals r ON r.principal_id = rm.role_principal_id +JOIN sys.database_principals m ON m.principal_id = rm.member_principal_id +WHERE m.type IN ('E', 'X'); +``` + +Expected: exactly the roles from step 6. A `db_owner` or `CONTROL` grant nobody asked for is the +usual leftover of a one-command tool. + +**3. On the cloud, the server side agrees.** + +```bash +az sql server ad-admin list -g -s -o table +az sql server show -g -n --query identity +``` + +Expected: the administrator from step 1, and a non-null identity if step 2 said you needed one. +Having run the command is not evidence it took. + +## References + +Each of the three files under `references/` is linked from the section that needs it, at the point +where the condition for opening it is stated. Tenant-side role assignment changes, so fetch the Learn +articles rather than recalling them. + +## Do not + +- Do not answer `Msg 33134` by editing the principal name. Check the server identity, its Graph + permission, and that the assign command used `-i`. +- Do not reach for `WITH OBJECT_ID` as a general fix. It answers `Msg 33131` and nothing else, and + wants the enterprise application object id, not the app registration's. +- Do not run `CREATE USER FROM EXTERNAL PROVIDER` on `master` and report the identity as configured. +- Do not attempt the lookup forms from a SQL-authenticated connection, server administrator included. + No grant makes those work. +- Do not transpose an authentication keyword between drivers, and never write + `ActiveDirectoryDefault` into an ODBC connection string. +- Do not use `Active Directory Password` in any spelling, or leave a .NET 7.0 application on an + `Active Directory *` mode without the extension package. +- Do not put a client secret in a connection string on Azure, or grant `CONTROL ON DATABASE` because + a tool did. Role design is `least-privilege-database-roles`. +- Do not enable Entra-only authentication on a shared server without naming what it turns off, or + assign one Azure Policy definition and call the estate enforced. +- Do not conclude a grant failed until the identity token cache has turned over. +- Do not answer `Msg 40530` by rewriting the batch. On a bare container it means the engine has no + Entra configuration. +- Do not read a clean container run as evidence the cloud path works. The server identity, `Msg + 33134` and Entra-only authentication have no local equivalent. diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/driver-auth-keywords.md b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/driver-auth-keywords.md new file mode 100644 index 0000000..4daf4f8 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/driver-auth-keywords.md @@ -0,0 +1,214 @@ +# The authentication keyword, per driver, verbatim + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [Microsoft.Data.SqlClient, for .NET](#microsoftdatasqlclient-for-net) +- [ODBC Driver 18 for SQL Server, including pyodbc](#odbc-driver-18-for-sql-server-including-pyodbc) +- [mssql-python](#mssql-python) +- [tedious, under the Node.js mssql package](#tedious-under-the-nodejs-mssql-package) +- [The Microsoft JDBC driver](#the-microsoft-jdbc-driver) +- [sqlcmd](#sqlcmd) +- [The same idea, five spellings](#the-same-idea-five-spellings) +- [Password authentication is deprecated everywhere](#password-authentication-is-deprecated-everywhere) +- [Sources](#sources) + +## How to use this file + +Read it before writing an authentication keyword into a connection string. Each list below is the +**complete** documented set for that driver, so a value not in the list is not a value. + +The failure this prevents is quiet: a driver that does not recognise the value may fall through to +a different mechanism and produce a login failure that reads as a database permission problem. + +## Microsoft.Data.SqlClient, for .NET + +The only family that writes the values **with spaces**. + +| Value | Since | +|---|---| +| `Active Directory Integrated` | 2.0.0 | +| `Active Directory Interactive` | 2.0.0 | +| `Active Directory Service Principal` | 2.0.0 | +| `Active Directory Device Code Flow` | 2.1.0 | +| `Active Directory Managed Identity`, and the alias `Active Directory MSI` | 2.1.0 | +| `Active Directory Default` | 3.0.0 | +| `Active Directory Workload Identity` | 5.2.0 | +| `Active Directory Password` | 1.0, deprecated | + +- User-assigned managed identity: `User Id=`. On 3.0 and later that is the **client id**; + on 2.1 it was the object id. +- `Active Directory Default` reads the same `User Id` property for the managed identity client id. +- `Active Directory Workload Identity` also takes the client id in `User Id`, and otherwise defaults + from environment variables. +- Microsoft warns that `Active Directory Default` "can come with performance impacts because it has + to look in multiple places for authentication information", and is not recommended where response + times are tight. Name the mode explicitly in production. + +**Version 7.0 is a breaking change.** The core package no longer carries the Azure and Entra +dependencies, and any of these modes needs the `Microsoft.Data.SqlClient.Extensions.Azure` package +added. Connection strings do not change. `SqlAuthenticationMethod.ActiveDirectoryPassword` became +`[Obsolete]` in the same release. An application that authenticated fine before the upgrade and +fails after it is usually missing that package reference, and says so at run time with +`Cannot find an authentication provider for 'ActiveDirectoryDefault'.` + +```bash +dotnet list package --include-transitive | grep Microsoft.Data.SqlClient +``` + +## ODBC Driver 18 for SQL Server, including pyodbc + +The complete documented set, verbatim from the keyword table: + +> `(not set)`(default), `(empty string)`, `SqlPassword`, `ActiveDirectoryIntegrated`, +> `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, `ActiveDirectoryServicePrincipal`, +> `ActiveDirectoryPassword` [DEPRECATED] + +**There is no `ActiveDirectoryDefault` and no `ActiveDirectoryManagedIdentity` in ODBC.** There is +also no device code mode. Writing any of those into a `pyodbc` connection string is the single most +common invented value in this area. + +Managed identity, quoted: + +> `ActiveDirectoryMsi` - Authenticate with a Microsoft Entra managed identity. For a user-assigned +> identity, set UID to the identity's client ID for Azure App Service or Azure Container Instance; +> otherwise, use its object ID. For system-assigned identity, UID isn't required. + +```text +server=;database=;Authentication=ActiveDirectoryMsi;Encrypt=yes; +server=;database=;UID=;Authentication=ActiveDirectoryMsi;Encrypt=yes; +``` + +Version notes: managed identity arrived in 17.3.1.1 for both identity kinds; +`ActiveDirectoryServicePrincipal` in 17.7, where `UID` is the client id and `PWD` the client secret. + +A token can be supplied instead through the `SQL_COPT_SS_ACCESS_TOKEN` pre-connection attribute. +When it is used, the connection string must not contain `UID`, `PWD`, `Authentication` or +`Trusted_Connection`. + +## mssql-python + +Seven modes, all through the `Authentication` connection-string keyword. + +| Value | What it does | +|---|---| +| `ActiveDirectoryDefault` | `DefaultAzureCredential`, which walks a chain of providers | +| `ActiveDirectoryInteractive` | Browser sign-in | +| `ActiveDirectoryDeviceCode` | Code entry on a second device | +| `ActiveDirectoryMSI` | Managed identity, either kind | +| `ActiveDirectoryServicePrincipal` | `UID` is the client id, `PWD` the client secret | +| `ActiveDirectoryIntegrated` | Kerberos, in a federated domain | +| `ActiveDirectoryPassword` | Deprecated | + +`ActiveDirectoryDefault`, `ActiveDirectoryInteractive` and `ActiveDirectoryDeviceCode` need the +`azure-identity` package installed alongside the driver. + +User-assigned managed identity, quoted: "Specify the client ID of a user-assigned managed identity +in the `UID` field." + +```python +"Server=.database.windows.net;" +"Database=;" +"Authentication=ActiveDirectoryMSI;" +"UID=;" +"Encrypt=yes;" +``` + +Note the spelling: `ActiveDirectoryMSI` here, `ActiveDirectoryMsi` in ODBC, and +`ActiveDirectoryManagedIdentity` in neither. + +## tedious, under the Node.js mssql package + +The value is an object, not a string, and the driver validates the type against a fixed list: +`default`, `ntlm`, `token-credential`, `azure-active-directory-password`, +`azure-active-directory-access-token`, `azure-active-directory-default`, +`azure-active-directory-msi-vm`, `azure-active-directory-msi-app-service`, +`azure-active-directory-service-principal-secret`. + +```js +authentication: { + type: 'azure-active-directory-default', + options: { clientId: '' } +} +``` + +**The client id is read from `authentication.options.clientId` and nowhere else.** A sample that +places `clientId` in the connection `options` bag beside `encrypt` appears to work only when the +matching environment variable is also set, which the credential chain picks up on its own. Remove +the environment variable and the same code silently falls back to the system-assigned identity. + +`token-credential` takes a credential object from the identity library directly, and is the +cleanest option when the credential is already constructed in application code. + +The Node.js package's own README lists an older, shorter set of types. It passes `authentication` +straight through, so the driver's list above is the authority. + +## The Microsoft JDBC driver + +`authentication=` accepts `NotSpecified` (the default), `SqlPassword`, `ActiveDirectoryPassword`, +`ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, `ActiveDirectoryServicePrincipal`, +`ActiveDirectoryServicePrincipalCertificate`, `ActiveDirectoryManagedIdentity` and +`ActiveDirectoryDefault`. + +- `ActiveDirectoryMSI` is accepted as a **synonym** for `ActiveDirectoryManagedIdentity`. Prefer the + long form on driver 12.2 and later. +- User-assigned client id goes in `msiClientId`, and from 12.2 the `user` property is also accepted. + JDBC takes the **client** id, unlike ODBC. +- Dependencies split by mode: the MSAL library for the interactive, integrated, password and service + principal modes, and the Azure identity library for the managed identity and default modes. + Missing the right one produces a class-loading failure rather than a login failure. + +## sqlcmd + +Verified by running the tool at version 1.10.0. Supplying an invalid value prints the accepted set: + +```text +Invalid federated authentication type 'NotARealMode': expected one of [ActiveDirectoryApplication +ActiveDirectoryServicePrincipal ActiveDirectoryDefault ActiveDirectoryIntegrated +ActiveDirectoryInteractive ActiveDirectoryManagedIdentity ActiveDirectoryMSI ActiveDirectoryPassword +ActiveDirectoryAzCli ActiveDirectoryDeviceCode ActiveDirectoryAzureDeveloperCli +ActiveDirectoryAzurePipelines ActiveDirectoryEnvironment ActiveDirectoryWorkloadIdentity +ActiveDirectoryClientAssertion ActiveDirectoryOnBehalfOf] +``` + +`-G` selects Entra authentication with the interactive or default flow, and +`--authentication-method=` names one of the above. Asking the tool is faster and more +reliable than recalling the list, and it is a good habit for any driver that will tell you. + +## The same idea, five spellings + +One concept, managed identity, across the drivers: + +| Driver | The value | +|---|---| +| `Microsoft.Data.SqlClient` | `Active Directory Managed Identity` | +| ODBC Driver 18 | `ActiveDirectoryMsi` | +| `mssql-python` | `ActiveDirectoryMSI` | +| `tedious` | `azure-active-directory-msi-app-service` or `azure-active-directory-msi-vm` | +| JDBC | `ActiveDirectoryManagedIdentity` | + +And the passwordless default mode does not exist at all in ODBC. + +## Password authentication is deprecated everywhere + +The drivers carry the same wording: + +> The ActiveDirectoryPassword authentication option (Microsoft Entra ID password authentication) is +> deprecated in the Microsoft SQL drivers. This high-risk authentication flow is incompatible with +> mandatory Microsoft Entra multifactor authentication (MFA) and might not work in tenants where MFA +> is enforced. Plan to migrate to a different Microsoft Entra authentication method. + +It rests on the resource owner password credentials grant. No removal version is stated anywhere, so +it is a deprecation rather than a removal, but a tenant enforcing multifactor authentication already +breaks it. The documented replacements: an interactive or integrated mode when a person is present, +managed identity on Azure, and a service principal with a certificate rather than a secret elsewhere. + +## Sources + +All read on 2026-08-27. Fetch rather than trust this summary when a value is load-bearing. + +- SqlClient: `/sql/connect/ado-net/sql/azure-active-directory-authentication`, which carries the value table, the version column and the 7.0 migration section. +- ODBC: `/sql/connect/odbc/using-azure-active-directory`, which carries the keyword table and the managed identity id rule. +- mssql-python: `/sql/connect/python/mssql-python/entra-authentication`, which carries the seven modes and the `UID` rule. +- JDBC: `/sql/connect/jdbc/connecting-using-azure-active-directory-authentication`, which carries the property values, the synonym and the dependency table. +- tedious: the driver's own source and released documentation, which is the authority over the wrapping package's README. diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/entra-only-and-policy.md b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/entra-only-and-policy.md new file mode 100644 index 0000000..66f35bd --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/entra-only-and-policy.md @@ -0,0 +1,140 @@ +# Microsoft Entra-only authentication, and enforcing it + +## Contents + +- [What enabling it actually does](#what-enabling-it-actually-does) +- [Order, and who is allowed to do it](#order-and-who-is-allowed-to-do-it) +- [What stops working](#what-stops-working) +- [The two Azure Policy definitions](#the-two-azure-policy-definitions) +- [Reading the policies rather than trusting a summary](#reading-the-policies-rather-than-trusting-a-summary) +- [Sources](#sources) + +## What enabling it actually does + +> When enabling Microsoft Entra-only authentication, SQL authentication is disabled at the server +> level and prevents any authentication based on any SQL authentication credentials. + +Three consequences worth being explicit about: + +- It applies to the **whole logical server**, so every database on it, and to the server + administrator login as well. +- Existing SQL logins and users are **not removed**. New ones can still be created by an Entra + account with the right permission. They simply cannot connect. +- Entra users with the right permission can still impersonate SQL users, and impersonation between + SQL authentication users keeps working while the feature is on. + +```bash +az sql server ad-only-auth enable -g -n +az sql server ad-only-auth disable -g -n +az sql server ad-only-auth get -g -n +``` + +```sql +SELECT SERVERPROPERTY('IsExternalAuthenticationOnly'); -- 1 enabled, 0 disabled +``` + +The CLI subcommands still carry the older Active Directory naming, which is expected: the product +name changed and the identifiers deliberately did not. + +## Order, and who is allowed to do it + +- **The Microsoft Entra administrator must be set before enabling.** Without one the setting stays + inactive, and enabling through the APIs fails as well. +- **The administrator cannot be removed while it is enabled.** Removing it through an API fails. + Removing the administrator and disabling the setting together is allowed with the right + permissions. + +The Azure roles are split on purpose, and neither of the obvious SQL roles can do the whole job. + +| Role | Set the Entra administrator | Toggle Entra-only | +|---|---|---| +| `SQL Server Contributor` | Yes | No | +| `SQL Security Manager` | No | Yes | +| `Contributor`, `Owner` | Yes | Yes | + +The rationale is separation of duties: a role that can create a server or set an administrator +should not also be able to turn a security control off. The actions behind the setting are +`Microsoft.Sql/servers/azureADOnlyAuthentications/*`, plus +`Microsoft.Sql/servers/administrators/read` for anyone who needs to see it in the resource pages. + +## What stops working + +In Azure SQL Database, with Entra-only authentication on: + +- Elastic jobs +- SQL Data Sync +- SQL Insights +- `EXEC AS` for Microsoft Entra group member accounts +- Change data capture across the boundary: a database created by an Entra user has CDC artifacts a + SQL user cannot change, and the reverse holds too +- Transactional replication into the database, because that path requires SQL authentication between + participants + +Check this list against what already runs on the server. Nothing warns at enable time. + +## The two Azure Policy definitions + +Both are built in, both default to the `Audit` effect, and both accept `Audit`, `Deny` and +`Disabled`. Read on 2026-08-27 from the live definitions. + +| Display name | Definition id | Evaluates | +|---|---|---| +| `Azure SQL logical servers should have Microsoft Entra-only authentication enabled during creation` | `abda6d70-9778-44e7-84a8-06713e6db027` | `Microsoft.Sql/servers` at create time | +| `Azure SQL Database should have Microsoft Entra-only authentication enabled` | `b3a22bc9-66de-45fb-98fa-00f5df42f41a` | `Microsoft.Sql/servers/azureADOnlyAuthentications` on an existing server | + +Their own descriptions say why one is not enough: + +> Require Azure SQL logical servers to be created with Microsoft Entra-only authentication. This +> policy doesn't block local authentication from being re-enabled on resources after create. + +> Require Azure SQL logical servers to use Microsoft Entra-only authentication. This policy doesn't +> block servers from being created with local authentication enabled. It does block local +> authentication from being enabled on resources after create. + +Both then say: "Consider using the 'Microsoft Entra-only authentication' initiative instead to +require both." + +The initiative is **`Azure SQL Database should have Microsoft Entra-only authentication`**, +`a55e4a7e-1b9c-43ef-b4b3-642f303804d6`, and it contains exactly those two definitions. + +Two details the definitions carry that a summary loses: + +- The creation-time policy **exempts a server whose `createMode` is `Restore`**, so a restored + server can arrive non-compliant without the assignment complaining at create time. +- Both exclude servers whose resource group is managed by the analytics workspace provider. + +A server can be created already compliant, and the administrator set in the same call: + +```bash +az sql server create -g -n -l \ + --enable-ad-only-auth \ + --external-admin-principal-type User \ + --external-admin-name "" \ + --external-admin-sid +``` + +## Reading the policies rather than trusting a summary + +The published article about these policies is older than the definitions and still names them with +the previous product wording, and it describes two definitions where the live catalog carries a +create-time and a steady-state pair per product plus an initiative. Reading the live definitions is +one command and settles it: + +```bash +az policy definition list \ + --query "[?displayName!=null && contains(displayName, 'Entra-only authentication')].{display:displayName, id:name, allowed:parameters.effect.allowedValues, def:parameters.effect.defaultValue}" \ + -o json + +az policy set-definition list \ + --query "[?displayName!=null && contains(displayName, 'Entra-only')].{display:displayName, id:name}" \ + -o json +``` + +Do the same before quoting a policy name in a plan. The names in this file were read that way, not +recalled, and they should be re-read rather than trusted after a few months. + +## Sources + +- Microsoft Entra-only authentication with Azure SQL: `/azure/azure-sql/database/authentication-azure-ad-only-authentication`, which carries the feature description, the permission split and the unsupported feature list. +- Azure Policy for Microsoft Entra-only authentication: `/azure/azure-sql/database/authentication-azure-ad-only-authentication-policy`, which carries the effects and the compliance behaviour, and whose policy names are behind the live catalog. +- The live definitions, read with Azure CLI 2.89.1 on 2026-08-27. diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/identity-errors.md b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/identity-errors.md new file mode 100644 index 0000000..d95c0b5 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth/references/identity-errors.md @@ -0,0 +1,230 @@ +# Identity errors, and the three forms of CREATE USER + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [The error numbers, with their real causes](#the-error-numbers-with-their-real-causes) +- [Msg 33134 in full](#msg-33134-in-full) +- [Msg 33131 and the OBJECT_ID form](#msg-33131-and-the-object_id-form) +- [Msg 40530, where the message and the cause disagree](#msg-40530-where-the-message-and-the-cause-disagree) +- [The no-lookup form, with worked T-SQL](#the-no-lookup-form-with-worked-t-sql) +- [The name to put in the brackets](#the-name-to-put-in-the-brackets) +- [Which id goes where](#which-id-goes-where) +- [What the server identity is for](#what-the-server-identity-is-for) +- [Verification queries](#verification-queries) +- [Sources](#sources) + +## How to use this file + +Read it when an error number needs its exact text and its actual cause, or when the plain +`CREATE USER [] FROM EXTERNAL PROVIDER` has failed and the next form has to be chosen +deliberately. Sources are listed at the end. Numbers that are not about identity live in +`diagnose-connection-errors`. + +## The error numbers, with their real causes + +| Number | What the text says | What it usually is | +|---|---|---| +| `33134` | The principal could not be resolved | The logical server has no identity, or that identity cannot read Microsoft Graph. Only ever seen when a service principal runs the statement | +| `33131` | The principal has a duplicate display name | Microsoft Entra ID allows two applications to share a display name and the engine requires a unique one. Answer with `WITH OBJECT_ID` and an alias | +| `37545` | The object id is not valid, or the caller lacks permission | The object id given to `WITH OBJECT_ID` does not exist in this tenant, or the wrong one of the two portal object ids was used | +| `18456` | `Login failed for user ''.` | The credential was evaluated and refused: the login does not exist, is disabled, or the secret is wrong | +| `18456` naming `` | Login failed for that literal user name | The token was accepted and no matching principal exists in the database. The database user was never created, or was created in the wrong database | +| `40530` | The `CREATE USER` statement must be the only statement in the batch | On an engine with no Microsoft Entra configuration, the refusal of `TYPE = E` and `TYPE = X`. The batch is not the cause. See below | +| `4060` | `Cannot open database "" requested by the login. The login failed.` | The login is valid and has no user in that database, or the database name is wrong. After a deployment, almost always a missing database user | + +`18456` and `4060` are answered here, identity-shaped or plain, because both arrive after the +credential was evaluated. `40532` reads identically and is the gateway refusing before a database +was reached, so it belongs to `diagnose-connection-errors`. + +## Msg 33134 in full + +```output +Msg 33134, Level 16, State 1, Line 1 +Principal 'test-user' could not be resolved. +Error message: 'Server identity is not configured. Please follow the steps in "Assign an Azure AD +identity to your server and add Directory Reader permission to your identity" +(https://aka.ms/sqlaadsetup)' +``` + +The mechanism, quoted: + +> When a Microsoft Entra user executes these commands, Azure SQL's Microsoft application uses +> delegated permissions to impersonate the signed-in user and queries Microsoft Graph using their +> permissions. This flow isn't possible with service principals, because an application can't +> impersonate another application. Instead, the SQL engine tries to use its server identity ... The +> server identity must exist and have the Microsoft Graph query permissions or the operations fail. + +So the same statement succeeds when a person runs it and fails from a pipeline, which is the part +that makes it look like a permissions bug in the pipeline's own credential. + +Assign the identity with the commands in the skill body. **The flag is `-i`, long form +`--assign_identity` with an underscore.** Measured 2026-09-03 on Azure CLI 2.90.0: the hyphenated +`--assign-identity` exits with `unrecognized arguments`, so a script carrying it assigns nothing and +the next `CREATE USER` raises `33134` again. + +The Graph side needs a `Privileged Role Administrator` and can only be done from a script. Grant +either the three application permissions, the least-privilege option, or the `Directory Readers` +role, which is broader than the server needs: + +- `User.Read.All` +- `GroupMember.Read.All` +- `Application.Read.All` + +For production, Microsoft recommends a role-assignable group holding `Directory Readers`, so a +group owner can add server identities without a `Privileged Role Administrator` in the loop for +each one. + +## Msg 33131 and the OBJECT_ID form + +```output +Msg 33131, Level 16, State 1, Line 4 +Principal 'myapp' has a duplicate display name. Make the display name unique in Azure Active +Directory and execute this statement again. +``` + +The documented answer: + +```sql +CREATE USER [] FROM EXTERNAL PROVIDER + WITH OBJECT_ID = ''; +``` + +Four rules the documentation is explicit about: + +1. **Only for nonunique names.** Learn calls it a troubleshooting repair item for nonunique service + principals, and says the plain statement should be used otherwise. +2. **Add a suffix to the name.** Without one the statement succeeds and nothing records which + principal it was for. The recommended shape is the original name plus the first five characters + of the object id, for example `myapp2ba6c`, and the alias must fit `sysname`, at most 128 + characters. +3. **The object id must exist in this tenant**, or the statement fails with + `Msg 37545, Level 16, State 1, Line 1 '' is not a valid object id for '' or you do not have + permission.` +4. **It is the enterprise application object id**, not the one shown on the app registration page. + The documentation carries a warning about exactly this pair, because both are labelled Object ID + and only one works. + +Naming an Azure resource the same as an existing app registration is one way to arrive here: two +principals then share a display name, and `Msg 33131` is the collision being reported. + +The display name in Microsoft Entra ID and the alias in the database are not synchronised in either +direction. Renaming one never changes the other. + +## Msg 40530, where the message and the cause disagree + +```output +Msg 40530, Level 16, State 1, Line 1 +The CREATE USER statement must be the only statement in the batch. +``` + +Measured 2026-09-03 on an Azure SQL Database container started with no `MSSQL_AAD_` variables, `EngineEdition` `5`, +`IsExternalAuthenticationOnly` `0`: a lone `CREATE USER [p5] WITH SID = 0x1111..., TYPE = E;` raises +it, `TYPE = X` raises it, `EXEC (@cmd)` and `GO` separation raise it, and a lone +`CREATE USER [p4] WITHOUT LOGIN;` succeeds. The batch is not the cause. The engine has no Microsoft +Entra configuration, refuses the external principal types, and reports a batch rule. + +**Microsoft Learn documents `40530` only as the batch rule**: the errors and events table gives +severity 16 and the text `The %.*ls statement must be the only statement in the batch.`, with no +documented connection to Microsoft Entra configuration. The observed behaviour does not match the +documented meaning. That is recorded here as a disagreement rather than resolved: either the engine +reuses a number for a cause it has no message of its own for, or the documentation is incomplete, +and neither has been confirmed with the product team. + +## The no-lookup form, with worked T-SQL + +Azure SQL Database accepts a form of `CREATE USER` that does not query Microsoft Graph at all, so it +needs no server identity and no Graph permission. It is the practical answer to `Msg 33134` when +tenant-level privilege is not available. Nothing validates the id, so a wrong value produces a user +that no token will ever match. + +One block covers all three principals: change the id and the `TYPE`, per the table below. `TYPE` is +`E` for a user, an application or a managed identity, and `X` for a group. + +```sql +DECLARE @principal_name SYSNAME = 'example-app'; +DECLARE @id UNIQUEIDENTIFIER = ''; +DECLARE @castId NVARCHAR(MAX) = CONVERT(VARCHAR(MAX), CONVERT (VARBINARY(16), @id), 1); +DECLARE @cmd NVARCHAR(MAX) = N'CREATE USER [' + @principal_name + '] WITH SID = ' + @castId + ', TYPE = E;'; +EXEC (@cmd); +``` + +**This form is cloud, or an Entra-configured engine, only.** On an Azure SQL Database container +started without the `MSSQL_AAD_` variables it raises `Msg 40530`, above, and nothing about the +statement can be changed to make it run. + +## The name to put in the brackets + +The engine matches the token against this name, so a wrong one produces `Msg 33134` or `Msg 33131` +and reads as a permissions problem. + +| Principal | The name | +|---|---| +| A person | Their user principal name | +| A group | Its display name | +| An app registration or a user-assigned managed identity | Its display name | +| A system-assigned managed identity | The name of the Azure resource that owns it | +| A system-assigned identity on a deployment slot | `/slots/` | + +The slot form is the one nobody guesses, and an application that works in production and fails in a +staging slot is usually missing that user. + +## Which id goes where + +Three different identifiers, all GUIDs, all called something ending in ID. + +| Clause or property | Principal | The id it wants | +|---|---|---| +| `WITH OBJECT_ID` | Service principal | The **object id** from the enterprise application, not the app registration | +| `WITH SID`, `TYPE = E` | User | The user's **object id** | +| `WITH SID`, `TYPE = E` | Service principal or managed identity | The **application (client) id** | +| `WITH SID`, `TYPE = X` | Group | The group's **object id** | +| Connection string, user-assigned identity | Managed identity | The **client id**, except ODBC, which wants the object id outside a small set of hosts | + +The `sid` read back from `sys.database_principals` for an application converts to its +**application id**, so it can be compared with the identity the workload actually runs as. + +## What the server identity is for + +It is not the application's identity and it is not needed for a person to create users. + +Learn: for Azure SQL Database the server identity is optional, and required only when a Microsoft +Entra service principal creates or manages Entra users, groups or applications on the server. + +Either a system-assigned or a user-assigned identity can serve. A user-assigned one can hold the +Graph permissions once and be shared across servers, and it is not deleted with the server. + +Deleting whichever identity is serving this role leaves the server unable to reach Microsoft Graph, +and Entra authentication fails until a replacement is assigned and granted. + +## Verification queries + +The two catalogue queries live in the skill body, under "Check it worked", and run against the user +database, never `master`. What they will not tell you: + +- `sid_bytes` is `16` for a contained database user, whose SID is the object or client id itself, + and `18` for a user created from a Microsoft Entra server login, whose SID carries an `AADE` + suffix. The two do not match, so a login-based user cannot be correlated to its login by SID until + that suffix is removed. +- A row that exists with the wrong `entra_id` is indistinguishable from success until the + application fails, so compare the value rather than checking that the row is there. +- On a container with no `MSSQL_AAD_` variables both queries return nothing, because no `E` or `X` + principal can be created there. + +```sql +SELECT SERVERPROPERTY('IsExternalAuthenticationOnly'); -- 1 on, 0 off +``` + +## Sources + +Fetch these rather than trusting this summary when the details matter. Read on 2026-08-27, except +the last, read 2026-09-03. + +- `/azure/azure-sql/database/authentication-aad-service-principal`: the `33134` text and the impersonation mechanism. +- `/azure/azure-sql/database/authentication-aad-service-principal-tutorial`: the Graph permission list, and the rule that only Entra users can create Entra users. +- `/sql/relational-databases/security/authentication-access/authentication-microsoft-entra-create-users-with-nonunique-names`: `33131`, `37545` and the `WITH OBJECT_ID` rules. +- `/sql/t-sql/statements/create-user-transact-sql`: the syntax, `SID` and `TYPE`, and the permission requirement. +- `/azure/azure-sql/database/authentication-azure-ad-user-assigned-managed-identity`: the server identity rules and the no-lookup escape. +- `/azure/azure-sql/database/authentication-aad-overview`: the `sys.database_principals` property table and the 16 against 18 byte SID difference. +- `/azure/azure-sql/database/authentication-azure-ad-logins`: what a SQL admin or SQL user cannot execute. +- `/sql/relational-databases/errors-events/database-engine-events-and-errors-31000-to-41399`: the `40530` row, documented only as the batch rule. diff --git a/plugins/microsoft-sql-ssms/skills/entra-id-auth/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/entra-id-auth/skill.spec.jsonc new file mode 100644 index 0000000..d39f617 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/entra-id-auth/skill.spec.jsonc @@ -0,0 +1,148 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "entra-id-auth", + "domain": "identity-and-access", + "value": ["multi-step-with-silent-failures", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize"], + "correction": "Agents present Microsoft Entra ID setup for an application identity as a linear sequence of commands that either succeed or report an error, when the sequence carries steps that succeed while achieving nothing and one error whose text names the wrong component. Msg 33134 reads 'Principal could not be resolved', which sends the agent to check the principal name, but the principal is almost always correct and the actual fault is that the logical server has no identity holding Microsoft Graph read permission. That fault only appears when a service principal rather than a signed-in user runs CREATE USER, so the same statement works by hand and fails in a pipeline. Agents also assume any administrator can create the Entra database user, when a SQL authenticated connection cannot create one at all, and that CREATE USER FROM EXTERNAL PROVIDER run against master helps an application that connects to a user database, when the statement adds the user to whichever database the connection is on and reports success either way. Finally they invent per driver authentication keywords, most often by writing ActiveDirectoryDefault into an ODBC connection string, which has no such value. The whole class surfaces later as a login failure, 18456 naming a token-identified principal or 4060 refusing the database the login asked for, and an agent reading those as a network or connection-string fault edits the one place that cannot fix them.", + "value_probe": { + "task": "I run the local Azure SQL Database container for development. I started it with nothing but the licence acceptance and an administrator password, no other environment variables.\n\nOur deployment pipeline creates database users for application identities, and I want to test that step locally before it runs for real. So I connected as the administrator and ran exactly this, on its own, as the only thing in the batch:\n\n```sql\nCREATE USER [orders-api] WITH SID = 0x28C1D9A6E63D4B4CA0BF25F0C97F86F3, TYPE = E;\n```\n\nThe engine refuses it, and the message tells me that this statement has to be the only statement in the batch. It is the only statement in the batch. I have tried it in a fresh session with nothing before it, I have tried separating everything with GO, and I have tried running it through EXEC with the text in a variable. Same refusal every time. A plain CREATE USER with a password is accepted in the same session, so my connection and my permissions are fine.\n\nWhat is this actually telling me, and what do I need to do to test this step?", + "gate": "The answer attributes the refusal to the engine having no Microsoft Entra configuration rather than to the shape or contents of the batch, and directs the reader either to configure the container for Microsoft Entra or to test this step against a database in Azure, instead of rewriting the statement.", + "states": [ + "The message text names the wrong cause; the batch is genuinely correct and no rearrangement of it will help.", + "The engine is refusing because it has no Microsoft Entra configuration, so it cannot create an external principal of any kind.", + "The acceptance of a password based user in the same session proves the permissions and the connection are not involved.", + "Testing this step requires either configuring the container for Microsoft Entra or running it against an engine that already has that configuration." + ], + "refutes": [ + "The refusal is about batch structure, so separating the statements with a batch separator, moving the statement into its own batch, or wrapping it in dynamic execution will resolve it.", + "The statement is syntactically wrong, the security identifier is malformed, or the wrong user type letter was used, and correcting the syntax will make it succeed.", + "The administrator connection lacks a permission needed to create this user, and granting a server level permission or connecting as a different administrator will fix it.", + "The identity named does not exist in the directory, so the engine could not resolve it and refused." + ] + }, + "posture": ["provision", "admin", "execute"], + // Raised to ga on 2026-09-21, and this comment replaces one that named a + // defect rather than a result. Measured 2026-09-20 at ten runs per arm. The + // gate, which asks whether the answer blames the missing Microsoft Entra + // configuration rather than the shape of the batch, reads 9 of 10 with the + // skill against 3 of 10 without. A sixty point gap is the largest single + // result in this catalog. + // + // The verdict used to miss it, and that was a defect in the rule rather than + // in this skill. eval/run-value-eval.mjs built its verdict from the states and + // refutes checks only, so the gate's own effect was computed, written into + // every report, and never read: the gate could subtract, by returning NO GATE + // when the skill-loaded arm failed it, and could never add. A skill whose whole + // measured effect was getting the model to the right approach could not score + // better than NEITHER however large the gap. That was fixed on 2026-09-21 and + // the rule now lives in eval/lib/verdict.mjs, where its header says what a + // decisive gate contributes and what it still cannot do on its own. + // + // RE-DERIVED FROM THE SAME STORED COUNTS, not re-run. The verdict is LIFT, the + // one statement sitting between the thresholds at 10 of 10 against 8 of 10 is + // still reported as open, and the gate speaks over it because it falls short + // in the skill's favour rather than against it. All five probes pass, so + // scripts/check-maturity-drift.mjs computes a ga ceiling and this field is now + // at it. + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-to-azure-sql", + "connect-from-dotnet", + "connect-from-python", + "connect-from-typescript-and-node", + "diagnose-connection-errors", + "provision-azure-sql-db", + "build-app-on-azure-sql", + "deploy-app-to-azure", + "least-privilege-database-roles", + "entra-logins-and-server-roles", + "managed-identity-across-azure-services", + "entra-groups-and-access-reviews", + "secure-by-default-checklist" + ], + "validation": { + "target": "both", + "setup": [ + "a scratch database on the substrate under test, and an administrative connection to it", + "for the cloud-only assertions below, a logical server whose Microsoft Entra administrator can be set, and an Azure resource carrying a managed identity or an app registration to stand in for the application" + ], + "assert": [ + "the Microsoft Entra administrator is read back from the server before any CREATE USER statement is attempted", + "the connection running CREATE USER FROM EXTERNAL PROVIDER is authenticated with Microsoft Entra ID, not with a SQL login, and is on the user database rather than master", + "Msg 33134 is answered by checking the logical server identity and its Microsoft Graph permission, not by editing the principal name", + "a wrong or unknown principal name produces Msg 33130, not 33134 and not 33131. Measured on two logical servers, 2026-09-05 and 2026-09-08, with an unknown address, a bare guid and a non-existent application name", + "a display name that is duplicated in the tenant produces Msg 33131, measured 2026-09-08 with CREATE USER [Microsoft Graph] FROM EXTERNAL PROVIDER", + "WITH OBJECT_ID is a modifier on FROM EXTERNAL PROVIDER and not an alternative to it: written alone the statement is Msg 37546, and with a guid absent from the tenant it is Msg 37545. Both measured 2026-09-08", + "if Microsoft Entra-only authentication is enabled, the administrator was set first and the features it disables were stated", + "a login failure is answered at the layer that raised it: 18456 as a login, a secret or a missing database user, and 4060 as a missing user in the database that was asked for, never as a firewall rule or a connection-string edit", + "on the container, an 18456 naming an empty user is routed to connect-to-azure-sql as missing MSSQL_AAD_ configuration rather than answered with a database user", + "on the container, Msg 40530 raised by a lone CREATE USER ... TYPE = E or TYPE = X is read as missing Microsoft Entra configuration on the engine, never as a batch that has to be rewritten" + ], + "probes": [ + { + "sql": "CREATE USER [probe_no_login] WITHOUT LOGIN;", + "expect": { "succeeds": true }, + "why": "The control the two probes below are read against, and the reason Msg 40530 cannot be believed. One CREATE USER statement, alone in a batch, accepted. Measured 2026-09-03 on a container started with no MSSQL_AAD_ variables: it creates a SQL_USER. Whatever refuses the next two, it is not the shape of the batch." + }, + { + "sql": "CREATE USER [probe_entra_app] WITH SID = 0x11112222333344445555666677778888, TYPE = E;", + "expect": { "error": 40530 }, + "why": "The no-lookup form the skill hands every pipeline that cannot get Microsoft Graph permission, refused on an engine with no Microsoft Entra configuration, and refused by a message naming the wrong cause: Msg 40530 is 'The CREATE USER statement must be the only statement in the batch', and this is the only statement in the batch. Measured 2026-09-03 with zero MSSQL_AAD_ variables set; EXEC (@cmd) and GO separation return 40530 as well, so no rewriting of the batch escapes it. The id round trip this probe once asserted is a cloud claim and needs an Entra-configured engine. If a later build accepts this statement, or refuses it with a number that names the missing configuration, the probe goes red and the skill should then say so." + }, + { + "sql": "CREATE USER [probe_entra_group] WITH SID = 0x11112222333344445555666677778888, TYPE = X;", + "expect": { "error": 40530 }, + "why": "TYPE = X is the group half of the same form, refused identically on 2026-09-03, which puts the refusal on the external principal types rather than on E alone. Pinning both spellings means a build that starts accepting either one is caught." + }, + { + "cmd": "az sql server update --help 2>&1 | grep -q -- '--assign_identity -i'", + "where": "host", + "expect": { "succeeds": true }, + "why": "Guards the flag spelling in the Msg 33134 section. Measured 2026-09-03 on Azure CLI 2.90.0: the assign-identity flag is -i, long form --assign_identity with an underscore, and the hyphenated --assign-identity that every other Azure CLI flag would lead an agent to write exits with 'unrecognized arguments'. A script carrying the hyphenated form assigns no server identity and the next CREATE USER raises 33134 again. If a future CLI accepts the hyphen this probe goes red and the skill should then say both work." + }, + { + "cmd": "sqlcmd -S 127.0.0.1,$PROBE_PORT -d $PROBE_DB --authentication-method=NotARealMode -Q 'SELECT 1' 2>&1 | grep -q ActiveDirectoryManagedIdentity", + "where": "host", + "expect": { "succeeds": true }, + "why": "The skill tells a reader to ask the driver for its accepted set rather than recall it, and this settles that sqlcmd will. Measured 2026-09-03 on sqlcmd 1.10.0: an invalid value is rejected before any connection is opened, printing the full accepted list, so the check needs no reachable server and no Microsoft Entra tenant. Grepping for ActiveDirectoryManagedIdentity also pins the one spelling the skill's per-driver table gives for sqlcmd." + } + ] + }, + "triggering": { + "explicit": [ + "Set up Microsoft Entra authentication for my Azure SQL database", + "Give my app a managed identity and let it connect to Azure SQL Database without a password", + "How do I create a database user for a managed identity in Azure SQL Database", + "Turn on Microsoft Entra-only authentication and enforce it with Azure Policy" + ], + "implicit": [ + "I get Principal could not be resolved when my pipeline runs CREATE USER", + "I set the MSSQL_AAD variables on my local container, how do I let my app sign in with Entra ID now", + "My app connects fine from my laptop but gets login failed for user token-identified principal once it is deployed", + "CREATE USER FROM EXTERNAL PROVIDER says there is a duplicate display name", + "I am the server admin and I still cannot create the Entra user", + "How do I stop putting the SQL password in my configuration", + "Which Authentication value do I put in the connection string for a user-assigned managed identity", + "Cannot open database requested by the login. The login failed", + "Login failed for user, what do I do" + ], + "routing": [ + "the grant step of an azd deployment, where the database side user and roles have to be created", + "a login failure whose user name is an identity rather than a SQL login", + "a decision between a system-assigned and a user-assigned managed identity for database access", + "the managed identity exists in Azure but the database says login failed" + ], + "negative": [ + "how do I connect to Azure SQL Database from Node.js", + "add retry logic and size the connection pool for my Azure SQL data access layer", + "what does error 40613 mean, my first connection failed", + "the pre-login handshake failed and the connection timed out", + "deploy my app to Azure with azd and provision the infrastructure", + "create an Azure SQL database and open the firewall for my address", + "design the least-privilege database roles my reporting service needs", + "create a SQL login and password for the local Azure SQL Database container", + "how do I sign in to the Azure portal" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/read-execution-plan b/plugins/microsoft-sql-ssms/skills/read-execution-plan deleted file mode 120000 index f1ab778..0000000 --- a/plugins/microsoft-sql-ssms/skills/read-execution-plan +++ /dev/null @@ -1 +0,0 @@ -../../../skills/read-execution-plan \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/read-execution-plan/SKILL.md b/plugins/microsoft-sql-ssms/skills/read-execution-plan/SKILL.md new file mode 100644 index 0000000..aeddf04 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/read-execution-plan/SKILL.md @@ -0,0 +1,216 @@ +--- +name: read-execution-plan +description: >- + Retrieves an Azure SQL Database execution plan, estimated or actual, and pulls out the + small set of facts that explain slowness: operators, estimated versus actual row counts, + warnings, missing-index hints, and the memory grant, instead of returning the whole plan + XML. Use when someone says "read this execution plan", "why did the optimizer choose a + scan here", "show me the actual plan not the estimated one", or hands over a plan and asks + what is wrong with it. Also covers retrieval itself: SET SHOWPLAN_XML, SET STATISTICS XML, + the plan-handle views, and Query Store, and the several ways a same-looking call returns + the wrong kind of plan, a stub, or NULL without raising an error. Not general slow-query + triage (diagnose-slow-query), lock analysis (diagnose-blocking-and-deadlocks), or + instance-level CPU or memory pressure (diagnose-resource-pressure). +--- + +# Read an Azure SQL Database execution plan + +Gets the plan that explains a query, in the smallest form that carries the signal, and tells +estimated numbers from measured ones. Measured 2026-08-29 against a live engine +reporting `EngineEdition` 5 and Edition `SQL Azure`; claims one session cannot measure are +attributed to Microsoft Learn inline. + +## Estimated is not actual, and the sources disagree about which you get + +| Source | Returns | Runs the query? | +|---|---|---| +| `SET SHOWPLAN_XML ON` | Estimated plan only | No | +| `SET STATISTICS XML ON` | Actual plan, `ActualRows` on every operator | Yes, for real | +| `sys.dm_exec_query_plan` | Estimated plan only, even after the query has run | No | +| `sys.dm_exec_query_plan_stats` | Last execution's actual plan | No, but off by default | +| `sys.dm_exec_query_statistics_xml` | Actual plan of a query running right now | No | +| `sys.query_store_plan` plus `sys.query_store_runtime_stats` | Actual plan and aggregated stats, persisted | No | + +`sys.dm_exec_query_plan` is the trap. It returns without error for a plan whose query has run +thousands of times, and the XML still carries zero occurrences of `ActualRows`. Reporting its +numbers as what happened is the wrong turn this skill exists for. + +## Retrieval, in order + +**1. Estimated plan, no execution.** A `SET SHOWPLAN` statement must be the only statement in +its batch, and cannot be used inside a stored procedure ([Learn](https://learn.microsoft.com/sql/t-sql/statements/set-showplan-xml-transact-sql)). +Pair it with anything else and the whole batch fails with `Msg 1067`, `The SET SHOWPLAN +statements must be the only statements in the batch`; nothing runs, not even the first half. + +```sql +SET SHOWPLAN_XML ON; +GO +SELECT status, COUNT(*) FROM dbo.orders WHERE status = 'open' GROUP BY status; +GO +SET SHOWPLAN_XML OFF; +GO +``` + +**2. Actual plan by re-running.** Same batch rule, and it executes the query for real. Never +wrap an insert, update, delete or merge in it expecting a dry run. + +```sql +SET STATISTICS XML ON; +GO +SELECT status, COUNT(*) FROM dbo.orders WHERE status = 'open' GROUP BY status; +GO +SET STATISTICS XML OFF; +GO +``` + +**3. Actual plan without re-running.** `sys.dm_exec_query_plan_stats` returns the last real +execution, and is silent about why it looks empty: the scoped configuration +`LAST_QUERY_PLAN_STATS` defaults to `OFF` ([Learn](https://learn.microsoft.com/sql/t-sql/statements/alter-database-scoped-configuration-transact-sql)), +and while it is off the view returns zero rows for every plan with no error. Turn it on once +per database, then look the plan up by object id rather than scanning the cache by text: + +```sql +ALTER DATABASE SCOPED CONFIGURATION SET LAST_QUERY_PLAN_STATS = ON; + +SELECT ps.execution_count, qp.query_plan +FROM sys.dm_exec_procedure_stats AS ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) AS qp +WHERE ps.object_id = OBJECT_ID('dbo.'); +``` + +**4. A query still running**, where there is no finished plan to fetch: + +```sql +SELECT r.session_id, qs.query_plan +FROM sys.dm_exec_requests AS r +CROSS APPLY sys.dm_exec_query_statistics_xml(r.session_id) AS qs +WHERE r.session_id <> @@SPID; +``` + +**5. Query Store**, for history and forced plans. Already on, and it cannot be turned off: +`ALTER DATABASE CURRENT SET QUERY_STORE = OFF` returns the warning `'QUERY_STORE=OFF' is +supported in this version of SQL Server` and leaves it running ([Learn](https://learn.microsoft.com/sql/relational-databases/performance/manage-the-query-store)). +Advice telling you to enable it was written for SQL Server, which also defaults +`QUERY_CAPTURE_MODE` to `ALL` where Azure SQL Database defaults to `AUTO`. Under `AUTO` a +query can be absent from `sys.query_store_query` after several executions: unresolved, never +proof it did not run, and not worth waiting on. Use step 3. Match on `object_id`, not a `LIKE` +search against the SQL you submitted, because `query_sql_text` stores the parameterized form: +a literal and a trailing comment came back here as +`(@1 tinyint)SELECT COUNT(*) FROM [dbo].[q] WHERE [id]>@1`. + +**6. Pull the small part, not the whole XML.** A two-operator plan measured here was 7.8 KB +and a two-table join 13 KB, both too large to paste into a response. Reduce on the server. +Every `.nodes()`, `.value()` and `.exist()` call needs `QUOTED_IDENTIFIER ON`, which a +client's default session may have off; without it the call fails with `Msg 1934`, an error +naming indexed views, computed columns, filtered indexes, query notifications and spatial +indexes, and never the setting that caused it. + +```sql +SET QUOTED_IDENTIFIER ON; +GO +;WITH XMLNAMESPACES (DEFAULT 'http://schemas.microsoft.com/sqlserver/2004/07/showplan') +SELECT n.value('@NodeId', 'int') AS NodeId, + n.value('@PhysicalOp', 'varchar(50)') AS PhysicalOp, + n.value('@EstimateRows', 'float') AS EstRows, + n.value('(.//RunTimeCountersPerThread/@ActualRows)[1]', 'float') AS ActualRows +FROM sys.dm_exec_procedure_stats AS ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) AS qp +CROSS APPLY qp.query_plan.nodes('//RelOp') AS t(n) +WHERE ps.object_id = OBJECT_ID('dbo.') +ORDER BY NodeId; +``` + +Open [references/dmv-and-xquery-reference.md](references/dmv-and-xquery-reference.md) before +writing the retrieval query, to get the plan-level half of this battery (warnings, +missing-index hint, memory grant) and what each plan XML attribute means. + +## Four ways a plan comes back empty or wrong, with no error + +- **`query_plan` is NULL at 128 levels of nesting.** The `xml` type's nesting limit means + `sys.dm_exec_query_plan` cannot return a deeply nested plan and returns NULL instead of + raising [error 6335](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-query-plan-transact-sql). + `sys.dm_exec_text_query_plan(plan_handle, 0, -1)` returns the same plan as text and has no + such limit. Reach for it whenever `query_plan` is NULL on a plan you know exists. +- **The plan is a stub.** For an ad hoc query using simple or forced parameterization, + `query_plan` holds the statement text and no operator tree. Fetch the plan handle of the + prepared parameterized query instead. +- **`query_plan_stats` returns a root node only.** For a query the engine judges simple, + typically OLTP shaped, it returns a Showplan holding just the `SELECT` node. No `RelOp` rows + is a real answer from a real plan, not a broken query. +- **The plan was evicted, or never cached.** Bulk operations and statements carrying string + literals over 8 KB are never cached, so no plan-handle route reaches them. + +Two permissions gate all of this and neither is implied by `SELECT`: the `SET` options need +`SHOWPLAN` on every database holding a referenced object, the views need `VIEW DATABASE +STATE`, and on Basic, S0, S1 and elastic pool databases they need `##MS_ServerStateReader##`. + +## Check it worked + +```sql +SELECT SESSIONPROPERTY('QUOTED_IDENTIFIER') AS quoted_identifier, + (SELECT value FROM sys.database_scoped_configurations + WHERE name = 'LAST_QUERY_PLAN_STATS') AS last_query_plan_stats; +``` + +Both must read `1`. A `0` in the first column is the `Msg 1934` coming; a `0` in the second is +step 3's empty result set, which you would misread as "no plan". + +Then confirm the plan is the kind you asked for. One string tells them apart, so ask the plan +rather than the view's name: + +```sql +SELECT CASE WHEN CAST(qp.query_plan AS nvarchar(max)) LIKE '%ActualRows%' + THEN 'actual' ELSE 'estimated' END AS plan_kind +FROM sys.dm_exec_procedure_stats AS ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) AS qp +WHERE ps.object_id = OBJECT_ID('dbo.'); +``` + +`estimated` means those row counts are guesses and every claim from them must say so. Zero +rows means step 3's configuration is still off, or the plan left the cache. + +If you do pull the whole document to a client, `xml` truncates at 256 characters by default +and a truncated plan parses as a corrupt one. Per `sqlcmd 1.10.0` help and +[Learn](https://learn.microsoft.com/sql/tools/sqlcmd/sqlcmd-utility), `-y` sets that width and +`0` removes the cap: + +```bash +sqlcmd -y 0 -Q "SELECT CAST(qp.query_plan AS nvarchar(max)) FROM sys.dm_exec_cached_plans cp +CROSS APPLY sys.dm_exec_query_plan(cp.plan_handle) qp WHERE cp.objtype = 'Proc'" +``` + +Step 6 avoids the question: the reduction happens on the server. + +## Reading what comes back + +- **`EstimateRows` against `ActualRows`, per operator.** A large gap means the estimate was + wrong there, from stale statistics, a correlated predicate, or a parameter-sensitive plan. + It is the most useful comparison in the document. +- **`Warnings`**, when present, names a real problem on that operator: a sort or hash spill, a + join with no equality predicate, an implicit conversion that changed the plan. Its absence + means none of those fired, not that the plan is fine. +- **`MissingIndexes`** is a single-query hint with an `Impact` percentage, blind to the rest + of the workload, indexes it would duplicate, and write cost. It is a lead. +- **`MemoryGrantInfo`** carries `GrantedMemory` against `MaxUsedMemory`. A wide gap is an + over-generous grant; `MaxUsedMemory` at or above `GrantedMemory` is spill risk even with no + `Warnings` element. + +## Do not + +- Do not report `EstimateRows` as what happened. Say "estimated" every time it is the only + number you have, and never present `sys.dm_exec_query_plan` output as the actual plan. +- Do not put a `SET SHOWPLAN` or `SET STATISTICS XML` statement in a batch with anything else, + or inside a stored procedure. The batch is rejected whole, not run in part. +- Do not run `SET STATISTICS XML ON` around a write expecting a dry run. It runs. +- Do not call `.nodes()`, `.value()` or `.exist()` before `SET QUOTED_IDENTIFIER ON`. +- Do not treat a `MissingIndexes` hint as instruction to create it. +- Do not paste the full plan XML into a response or into context "to be safe". +- Do not read a NULL `query_plan`, a root-only plan, or an absence from Query Store as an + error or as proof the query never ran. Each has a documented cause above, and none of them + is fixed by waiting. + +## References + +- [references/dmv-and-xquery-reference.md](references/dmv-and-xquery-reference.md), when you + need the full retrieval query for a source in the table above, the rest of the XQuery + battery, or the meaning of a plan XML attribute you are about to quote. diff --git a/plugins/microsoft-sql-ssms/skills/read-execution-plan/references/dmv-and-xquery-reference.md b/plugins/microsoft-sql-ssms/skills/read-execution-plan/references/dmv-and-xquery-reference.md new file mode 100644 index 0000000..0a11d22 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/read-execution-plan/references/dmv-and-xquery-reference.md @@ -0,0 +1,204 @@ +# DMV and XQuery reference for execution plans + +## Contents + +- [Retrieval query for each source](#retrieval-query-for-each-source) +- [When the plan comes back NULL, a stub, or a root node only](#when-the-plan-comes-back-null-a-stub-or-a-root-node-only) +- [Who is allowed to read a plan](#who-is-allowed-to-read-a-plan) +- [The XQuery battery](#the-xquery-battery) +- [Plan XML attributes, field by field](#plan-xml-attributes-field-by-field) +- [Sizes actually measured](#sizes-actually-measured) + +## Retrieval query for each source + +Every query below was run against a table of tens of thousands of rows with an index that +did not cover the filtered column, on this engine. + +**Estimated or actual by re-running.** This is the query the sizes table at the end was +measured against. Swap `SHOWPLAN_XML` for `STATISTICS XML` to get the actual plan; the +one-statement-per-batch rule and the `Msg 1067` on breaking it are identical for both: + +```sql +SET SHOWPLAN_XML ON; +GO +SELECT customer_id, COUNT(*) AS n, SUM(amount) AS total +FROM dbo.orders +WHERE status = 'open' +GROUP BY customer_id +HAVING COUNT(*) > 5 +ORDER BY total DESC; +GO +SET SHOWPLAN_XML OFF; +GO +``` + +**Actual plan of a query already run, by object.** Turn the scoped configuration on once per +database, then look the plan up by object id, not by scanning the cache. On a busy instance a +`sys.dm_exec_query_stats` join to `sys.dm_exec_sql_text` filtered with `LIKE` across every +cached plan took several minutes; filtered by `object_id` it returned in under a second: + +```sql +ALTER DATABASE SCOPED CONFIGURATION SET LAST_QUERY_PLAN_STATS = ON; + +SELECT + ps.execution_count, + ps.total_logical_reads / ps.execution_count AS avg_logical_reads, + qp.query_plan +FROM sys.dm_exec_procedure_stats ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) qp +WHERE ps.object_id = OBJECT_ID('dbo.'); +``` + +Confirmed on this engine: `sys.dm_exec_query_plan(ps.plan_handle)` in the same shape returns a +plan with zero occurrences of `ActualRows` for the same already-executed procedure. +`sys.dm_exec_query_plan_stats` in its place returned `ActualRows` on every operator, and only +after the scoped configuration above was on; before that, zero rows and no error. + +**Query Store**, for a persisted history. Prefer matching on `object_id` (join +`sys.query_store_query` to the procedure or module, if there is one) over a `LIKE` search on +`query_sql_text`, for the reason given below: + +```sql +SELECT qsq.query_id, qsqt.query_sql_text, qsrs.count_executions, qsrs.avg_duration, + qsrs.avg_logical_io_reads, qsp.query_plan, qsp.is_forced_plan +FROM sys.query_store_query qsq +JOIN sys.query_store_query_text qsqt ON qsq.query_text_id = qsqt.query_text_id +JOIN sys.query_store_plan qsp ON qsp.query_id = qsq.query_id +JOIN sys.query_store_runtime_stats qsrs ON qsrs.plan_id = qsp.plan_id +WHERE qsq.object_id = OBJECT_ID('dbo.'); +``` + +`sys.database_query_store_options.actual_state_desc` read `READ_WRITE` on a freshly created +database with nothing configured, so this engine ships with Query Store on, and +`query_capture_mode_desc` read `AUTO`. Learn documents `AUTO` as the SQL Database default and +`ALL` as the SQL Server default, and says Query Store cannot be turned off here at all: +`SET QUERY_STORE = OFF` returns a warning and leaves it running. + +**What was actually observed about capture timing, as data points, not a rule.** A query run +three times was absent from `sys.query_store_query` immediately and again eight seconds later, +and present about fifteen minutes later, after dozens of unrelated batches from other sessions +on this shared instance. Execution count, elapsed time and instance activity were never +isolated, so that does not show waiting closed the gap. A second, tighter check on the same +engine ran a simpler auto-parameterizable query 4 times, then 30 times, with an explicit +`sp_query_store_flush_db` in between, and it stayed absent throughout. The two do not agree, +and neither pins the mechanism down. **Treat absence from Query Store as unresolved, never as +proof the query did not run, and do not wait on it or retry on a timer**; the plan-handle route +above does not depend on the capture policy at all. + +**A captured query can still look absent to a `LIKE` search.** `query_sql_text` stores the +parameterized form, not the text submitted. Here, a query with a literal comparison value and +a trailing comment was stored as: + +``` +(@1 tinyint)SELECT COUNT(*) FROM [dbo].[q] WHERE [id]>@1 +``` + +with the comment gone and the literal replaced by a parameter. A `LIKE` search for the literal +or the comment returns zero rows for a query that is genuinely captured. Match on `object_id`, +as above, or on the parameterized shape you expect, not the string you submitted. + +## When the plan comes back NULL, a stub, or a root node only + +Four silent outcomes, all documented on the Learn pages for +[sys.dm_exec_query_plan](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-query-plan-transact-sql) +and +[sys.dm_exec_query_plan_stats](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-objects/sys-dm-exec-query-plan-stats-transact-sql). + +| What you see | Cause | Instead | +|---|---|---| +| `query_plan` is `NULL` | The plan meets 128 levels of nested elements, past what the `xml` type holds. Older engines raised error 6335; current ones return `NULL` | `sys.dm_exec_text_query_plan`, below: `nvarchar(max)`, no nesting limit | +| `query_plan` is `NULL` | Evicted, or never cacheable: bulk operations, and statements carrying string literals over 8 KB | Re-run under `SET STATISTICS XML ON`; the cache has nothing to give | +| `query_plan` holds statement text, no operator tree | An ad hoc query that used simple or forced parameterization; the plan sits under the prepared statement | Fetch the prepared query's plan handle, not the ad hoc one | +| `_plan_stats` returns a root node and no `RelOp` | The engine judged the query simple, typically OLTP shaped, and returned a simplified Showplan | Nothing is broken. For the operator tree, re-run under `SET STATISTICS XML ON` | + +The text form of the same plan. `0, -1` asks for the whole batch rather than one statement's +offsets: + +```sql +SELECT ps.execution_count, tqp.query_plan +FROM sys.dm_exec_procedure_stats ps +CROSS APPLY sys.dm_exec_text_query_plan(ps.plan_handle, 0, -1) tqp +WHERE ps.object_id = OBJECT_ID('dbo.'); +``` + +## Who is allowed to read a plan + +Neither is implied by `SELECT`, and this failure is loud: a permission error, not an empty +result. + +| To do this | You need | +|---|---| +| `SET SHOWPLAN_XML`, `SET STATISTICS XML`, `SET STATISTICS PROFILE` | `SHOWPLAN` on every database holding an object the statement references: `GRANT SHOWPLAN TO [user];` | +| `sys.dm_exec_query_plan`, `_plan_stats`, `_statistics_xml`, `_text_query_plan` | `VIEW DATABASE STATE`, or membership in the `##MS_ServerStateReader##` server role | +| The same views on a Basic, S0, S1 or elastic pool database | `VIEW DATABASE STATE` is not enough. Server admin, Microsoft Entra admin, or `##MS_ServerStateReader##` | + +## The XQuery battery + +Querying the `xml`-typed plan column directly returns only the fields that carry signal. +Requires `QUOTED_IDENTIFIER ON`: confirmed on this engine's default client session, +`SESSIONPROPERTY('QUOTED_IDENTIFIER')` read `0`, and `.nodes()`, `.value()` and `.exist()` then +all fail with `Msg 1934`. + +Per-operator estimate versus actual, in one round trip: + +```sql +SET QUOTED_IDENTIFIER ON; +GO +;WITH XMLNAMESPACES (DEFAULT 'http://schemas.microsoft.com/sqlserver/2004/07/showplan') +SELECT + n.value('@NodeId', 'int') AS NodeId, + n.value('@PhysicalOp', 'varchar(50)') AS PhysicalOp, + n.value('@EstimateRows', 'float') AS EstRows, + n.value('(.//RunTimeCountersPerThread/@ActualRows)[1]', 'float') AS ActualRows, + n.value('(.//Warnings)[1]', 'varchar(200)') AS WarningPresent +FROM sys.dm_exec_procedure_stats ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) qp +CROSS APPLY qp.query_plan.nodes('//RelOp') AS t(n) +WHERE ps.object_id = OBJECT_ID('dbo.') +ORDER BY NodeId; +GO +``` + +Plan-level summary, in the same round trip shape: + +```sql +;WITH XMLNAMESPACES (DEFAULT 'http://schemas.microsoft.com/sqlserver/2004/07/showplan') +SELECT + qp.query_plan.exist('//Warnings') AS HasWarnings, + qp.query_plan.exist('//MissingIndexes') AS HasMissingIndex, + qp.query_plan.value('(//MemoryGrantInfo/@GrantedMemory)[1]', 'int') AS GrantedMemoryKB +FROM sys.dm_exec_procedure_stats ps +CROSS APPLY sys.dm_exec_query_plan_stats(ps.plan_handle) qp +WHERE ps.object_id = OBJECT_ID('dbo.'); +``` + +Run against a two-table join with a filter, an aggregate and a sort, on this engine, the +first query returned 6 rows and the second returned a single row reading `HasWarnings = 0`, +`HasMissingIndex = 1`, `GrantedMemoryKB = 2672`. Combined, both results ran under 850 +characters, against a full plan document of roughly 13,150 characters for the same plan, +about six percent of the size. + +## Plan XML attributes, field by field + +| Attribute | Where | Meaning | +|---|---|---| +| `PhysicalOp` / `LogicalOp` | `RelOp` | The operator the engine chose, and the relational operation it implements. When these differ (a `Hash Match` implementing an `Aggregate`, for example) the physical choice is the one to question. | +| `EstimateRows` | `RelOp` | Compile-time row estimate for that operator. Present on every plan, estimated or actual. | +| `ActualRows` | `RunTimeCountersPerThread`, nested under `RelOp` | Only present on an actual plan. A wide gap against `EstimateRows` on the same operator is the strongest single signal in the document. | +| `Warnings` | Child of `RelOp`, only present when triggered | Spills, missing join predicates, and plan-affecting implicit conversions surface here. Its absence, on the plans measured for this skill, meant none of those conditions fired; it does not mean the plan is optimal. | +| `MissingIndexes` / `MissingIndexGroup` `Impact` | Under `QueryPlan`, sibling of the operator tree | A single-query suggestion with an estimated percentage improvement. It has no visibility into the rest of the workload or existing indexes. | +| `MemoryGrantInfo` `GrantedMemory` / `MaxUsedMemory` | Under `QueryPlan` | Granted and actually used memory, in kilobytes. `MaxUsedMemory` approaching or exceeding `GrantedMemory` signals spill risk even with no `Warnings` element present. | +| `IsMemoryGrantFeedbackAdjusted` | `MemoryGrantInfo` | Whether the engine already adjusted this grant from a previous execution's feedback. Read as `"No: Accurate Grant"` on a first-time plan measured here. | + +## Sizes actually measured + +| Query shape | Source | Size | +|---|---|---| +| Single table, filter, aggregate, sort | Estimated (`SHOWPLAN_XML`) | 7,816 characters | +| Single table, filter, aggregate, sort | Actual (`STATISTICS XML`) | 9,758 characters | +| Two-table join, filter, aggregate, sort | Actual, via `dm_exec_query_plan_stats` | 13,150 characters | +| Same two-table join | The XQuery battery result, both queries combined | Under 850 characters | + +A client truncates `nvarchar(max)` and `xml` output to a fixed display width, so the full +document needs that turned off separately. The XQuery battery avoids the question: the +reduction happens on the server and only the small result crosses the wire. diff --git a/plugins/microsoft-sql-ssms/skills/read-execution-plan/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/read-execution-plan/skill.spec.jsonc new file mode 100644 index 0000000..2bed21a --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/read-execution-plan/skill.spec.jsonc @@ -0,0 +1,179 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "read-execution-plan", + "domain": "performance-and-diagnostics", + "value": ["not-in-training-data", "multi-step-with-silent-failures"], + "correction": "Agents asked to read a query's execution plan reach for sys.dm_exec_query_plan or a first look at Query Store and treat what comes back as what the query actually did, when sys.dm_exec_query_plan only ever returns the compile-time estimated plan with no ActualRows attribute on any operator, and Query Store's default AUTO capture policy can leave a query absent from sys.query_store_query after several executions in a row, so the agent reports an estimate as a measurement, or treats a query's absence from Query Store as proof it never ran, when the absence is inconclusive and a plan-handle-based source answers the question directly instead of waiting on it.", + "value_probe": { + "gate": "The answer refuses the plan already in hand as a record of what the procedure did, because it carries only estimates, and routes to a source whose operators carry the counts the query really processed without re-running a procedure that writes, or says plainly that re-running is the only option left and why. Treating the retrieved plan as the measurement has not reached it.", + "task": "A nightly stored procedure dbo.rebuild_summary on Azure SQL Database takes forty minutes on some nights and two minutes on others. It ran tonight and it was one of the slow ones. I do not want to run it again, it writes to real tables that the morning reports read.\n\nI want to see, operator by operator, what it actually did on that run, and in particular where the row counts it expected are far from what it really processed.\n\nI already have this back:\n\n```sql\nSELECT qp.query_plan\nFROM sys.dm_exec_procedure_stats AS ps\nCROSS APPLY sys.dm_exec_query_plan(ps.plan_handle) AS qp\nWHERE ps.object_id = OBJECT_ID('dbo.rebuild_summary');\n```\n\nIt returned a plan, so I assume that is the run I care about. I also looked in sys.query_store_query for the statements and several of them are not there at all, which worried me because they definitely ran.\n\nDoes what I have answer the question, and if not, what are the exact steps to get what I need?", + // 2026-09-19: the second statement used to read "There is a source that returns the last real + // execution's plan without running the query again, and it depends on a database setting that + // is off unless somebody turned it on." It measured the route to the actual plan, which the + // model already reaches 8 of 10 times with no skill loaded, so it had no room to show the + // skill adding anything and could only fall. That route is still enforced, by the gate, which + // requires the answer to reach a source whose operators carry the real counts. The + // replacement measures the step after it, reading the plan operator by operator: none of the + // ten no-skill answers named the session setting those XML calls need, and all ten + // skill-loaded answers did. + "states": [ + "The plan already retrieved carries only the counts the optimizer estimated and no record of what the query processed, so it cannot answer the question as asked.", + "Reading the plan operator by operator means querying the plan XML, and those calls are refused unless QUOTED_IDENTIFIER is ON for the session, with a message that names other features and never the setting.", + "A statement missing from the query history is inconclusive rather than proof that it did not run, because the default capture policy does not record every query.", + "Re-running the procedure to get the plan would perform its writes for real, so it is not a dry run." + ], + "refutes": [ + "sys.dm_exec_query_plan returns the row counts the query actually processed once the query has executed, so the plan already in hand is the measurement.", + "A statement absent from the query history proves it never executed, or proves the history feature is switched off on this database.", + "The query history has to be enabled on an Azure SQL Database before it captures anything, and it can be switched off again afterwards.", + "The only way to obtain a plan carrying real row counts is to run the query again with the actual plan setting turned on.", + "The estimated and actual row counts already present in that plan can be compared against each other to find the misestimate." + ] + }, + "posture": ["read", "inspect", "admin"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "diagnose-slow-query", + "diagnose-resource-pressure", + "diagnose-blocking-and-deadlocks", + "capture-with-extended-events" + ], + "validation": { + "target": "container", + "setup": [ + "a table with a few thousand rows and a predicate column that carries no index", + "a stored procedure wrapping a query against that table so its plan can be looked up by object id", + "the database's default LAST_QUERY_PLAN_STATS scoped configuration, left at its default OFF" + ], + "assert": [ + "the estimated plan for a query with an unindexed equality or range predicate carries a MissingIndexes recommendation naming that column", + "sys.dm_exec_query_plan_stats returns zero rows for a plan until LAST_QUERY_PLAN_STATS is turned on for the database, and returns the actual plan with ActualRows populated once it is", + "a query executed only two or three times does not appear in sys.query_store_query immediately afterward, under the database's default AUTO capture policy", + "sys.query_store_query_text.query_sql_text stores the engine's parameterized form of a query rather than the literal text submitted, so a LIKE search against a literal predicate value or a comment can return zero rows for a query that is genuinely captured", + "sys.dm_exec_query_plan returns query_plan NULL, and raises no error, for a plan at or past 128 levels of nested xml elements, where sys.dm_exec_text_query_plan returns the same plan as text: not probed here because building a plan that deep is out of reach for one batch, sourced to Microsoft Learn", + "sys.dm_exec_query_plan_stats returns a simplified Showplan carrying only the root SELECT node, with no RelOp elements, for a query the engine judges simple: not probed here because the threshold is internal and undocumented, sourced to Microsoft Learn" + ], + "probes": [ + { + "sql": "SET SHOWPLAN_XML ON;\nSELECT 1 AS never_runs;", + "expect": { + "error": 1067 + }, + "why": "The skill's first retrieval step turns on a rule that has no partial failure: a SET SHOWPLAN statement sharing a batch with anything else takes the whole batch down, so the SELECT never runs and no plan comes back either. Msg 1067, 'The SET SHOWPLAN statements must be the only statements in the batch', from the Microsoft Learn error table for 1000 to 1999. An agent that writes the SET and the query into one round trip gets nothing and has no plan to read." + }, + { + "sql": "ALTER DATABASE CURRENT SET QUERY_STORE = OFF;\nSELECT actual_state_desc FROM sys.database_query_store_options;", + "expect": { + "rows": [ + [ + "READ_WRITE" + ] + ] + }, + "why": "Advice written for SQL Server tells a reader to enable Query Store. On Azure SQL Database it is already on and cannot be turned off: Microsoft Learn's Query Store management page states that SET QUERY_STORE = OFF returns the warning 'QUERY_STORE=OFF' is not supported in this version of SQL Server. A warning, not an error, so the statement completes and the state is unchanged. If this ever starts failing or ever actually turns Query Store off, step 5 of the skill is describing a different engine." + }, + { + "sql": "CREATE USER probe_plan_reader WITHOUT LOGIN;\nEXECUTE AS USER = 'probe_plan_reader';\nSELECT HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'SHOWPLAN') AS has_showplan,\n HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'VIEW DATABASE STATE') AS has_view_database_state;\nREVERT;", + "expect": { + "rows": [ + [ + 0, + 0 + ] + ] + }, + "why": "Every route in this skill needs a permission that SELECT does not imply: SHOWPLAN for the SET options and VIEW DATABASE STATE for the plan-handle views, per the Microsoft Learn permissions sections of SET SHOWPLAN_XML and sys.dm_exec_query_plan. A plain database user has neither, so an agent that writes a working retrieval query for an application login still gets nothing back. Both names resolving to 0 rather than NULL also proves they are real database-scoped permissions on this engine." + }, + { + "sql": "CREATE TABLE dbo.probe_plan_text_orders (id int NOT NULL PRIMARY KEY, status nvarchar(20) NOT NULL);\nINSERT INTO dbo.probe_plan_text_orders (id, status)\n SELECT TOP (500) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)), 'open'\n FROM sys.all_objects AS a CROSS JOIN sys.all_objects AS b;\nEXEC sys.sp_executesql N'CREATE PROCEDURE dbo.probe_plan_text_proc AS BEGIN DECLARE @n int; SELECT @n = COUNT(*) FROM dbo.probe_plan_text_orders WHERE status = ''open''; END';\nEXEC dbo.probe_plan_text_proc;\nSELECT COUNT(*) AS text_plans_with_operators\n FROM sys.dm_exec_procedure_stats AS ps\n CROSS APPLY sys.dm_exec_text_query_plan(ps.plan_handle, 0, -1) AS tqp\n WHERE ps.object_id = OBJECT_ID('dbo.probe_plan_text_proc')\n AND tqp.query_plan LIKE '%';\nSELECT @x.value('(/a/@b)[1]', 'int') AS v;", + "expect": { + "error": 1934 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Every XML method call needs QUOTED_IDENTIFIER ON, and the message names indexed views, computed columns and spatial indexes and never the setting that caused it, which is why this skill checks the setting rather than reading the message. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SET QUOTED_IDENTIFIER ON;\nDECLARE @x xml = N'';\nSELECT @x.value('(/a/@b)[1]', 'int') AS v;", + "expect": { + "succeeds": true + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The control for the probe above. Without it a red there could mean XML methods do not work at all rather than that one session setting decides it. It goes red if the statement raises." + } + ] + }, + "triggering": { + "explicit": [ + "Read this execution plan and tell me what's expensive", + "Get me the actual execution plan for this query, not the estimated one", + "Show me the execution plan and explain why it picked a hash join" + ], + "implicit": [ + "Here's a query plan XML, what's the one thing wrong with it", + "The optimizer keeps choosing a table scan here and I don't understand why", + "I ran this query and now I want to see what it actually did operator by operator" + ], + "negative": [ + "My query is slow, find out why", + "Sessions are blocking each other, what is holding the lock", + "CPU on the database is pinned near 100 percent, what is using it", + "Capture what queries ran over the last five minutes", + "Write a query that returns the top customers by order total" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/restore-and-recover b/plugins/microsoft-sql-ssms/skills/restore-and-recover deleted file mode 120000 index 4ef369b..0000000 --- a/plugins/microsoft-sql-ssms/skills/restore-and-recover +++ /dev/null @@ -1 +0,0 @@ -../../../skills/restore-and-recover \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/restore-and-recover/SKILL.md b/plugins/microsoft-sql-ssms/skills/restore-and-recover/SKILL.md new file mode 100644 index 0000000..425d313 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/restore-and-recover/SKILL.md @@ -0,0 +1,240 @@ +--- +name: restore-and-recover +description: >- + Recovers an Azure SQL Database after data loss, an accidental drop or a bad deployment, using + point-in-time restore, geo-restore and long-term retention. Use when someone asks to "restore + my Azure SQL database", "undo a dropped table or database", "roll back to before this migration + ran", "recover from a region outage", or asks for RESTORE DATABASE or BACKUP DATABASE syntax. + There is no backup or restore T-SQL here: BACKUP DATABASE, RESTORE DATABASE and every RESTORE + ... ONLY variant are refused, and restoring is a control plane operation. Every restore creates + a new database beside the one being recovered rather than overwriting it, so this covers the + restore type and the rename or connection swap back. Not a schema rollback + (schema-migrations-safely), not a logical export or import (sqlpackage-import-export), not + creating it (provision-azure-sql-db). +--- + +# Recover an Azure SQL Database + +Engine behaviour measured 2026-08-29 and 2026-09-03 against an engine reporting +`SERVERPROPERTY('EngineEdition')` 5, Edition `SQL Azure`, `ProductVersion` 12.0.2000.8. Every command line was taken on 2026-09-03 from +the help output of `az` 2.90.0 and SqlPackage 170.4.83.3 on this machine; the scope and retention +limits come from the Microsoft Learn pages in References. The restores need a real logical +server and were not run, so resource names are placeholders. + +## The correction + +Asked to recover an Azure SQL Database, an agent trained on the boxed engine reaches for +`RESTORE DATABASE ... WITH REPLACE`, `BACKUP DATABASE`, `RESTORE HEADERONLY` or +`ALTER DATABASE ... SET RECOVERY`. None of that surface exists here. Learn is blunt: you recover +using the portal, PowerShell, the Azure CLI or the REST API, and you can't use Transact-SQL. The +second wrong belief compounds the first. Learn again: **you can't overwrite an existing database +during restore.** Every restore creates a new database under the name you pass, and moving it to +where the application connects is a step you perform yourself. Nothing warns you if you skip it, +so the failure surfaces later as a connection error blamed on the application. + +## 1. There is no restore T-SQL, measured + +| Written from habit | What happens here | +|---|---| +| `RESTORE DATABASE db FROM DISK = '...' WITH REPLACE` | `Msg 40510`, not supported | +| `BACKUP DATABASE db TO DISK = '...'` or `TO URL = '...'` | `Msg 40510`, not supported | +| `BACKUP LOG db TO DISK = '...'` | `Msg 40510`, not supported | +| `RESTORE HEADERONLY` / `FILELISTONLY` / `VERIFYONLY` | `Msg 40510`, and the text names `RESTORE VOLUME` | +| `EXEC sys.sp_get_database_backup_policy` | `Msg 2812`, could not find stored procedure | +| `ALTER DATABASE db SET RECOVERY FULL` / `SIMPLE` | `Msg 40517`, keyword or option not supported | + +There is no exception in this family, and the two numbers are the parser's own distinction: +`Msg 40510` rejects a whole statement, `Msg 40517` rejects one option of a statement that is +otherwise supported, which is why `ALTER DATABASE` still renames a database in section 5. Every +row is a signal the agent is solving the wrong problem. + +## 2. Find the restore point before you name it + +`--time` must be at or after the source's `earliestRestoreDate`, so read it: + +```bash +az sql db show --resource-group --server --name \ + --query earliestRestoreDate --output tsv +``` + +A dropped database is not in `az sql db show` at all. It is a separate resource carrying the +deletion timestamp you need, and its `name` is a composite, so read `databaseName`: + +```bash +az sql db list-deleted --resource-group --server \ + --query "[].{db:databaseName, deleted:deletionDate, earliest:earliestRestoreDate}" \ + --output table +``` + +## 3. Three restores, three commands, and they do not have the same reach + +Point-in-time, into a new name on the same server. Pass the source's own tier so the copy can +stand in for it: + +```bash +az sql db restore --resource-group --server --name \ + --dest-name -restored --time "2026-09-02T14:30:00" \ + --edition GeneralPurpose --service-objective GP_Gen5_2 --backup-storage-redundancy Geo +``` + +A dropped database uses the same command with `--deleted-time` in place of, or alongside, `--time`, +matching `deletionDate` from the listing above exactly: + +```bash +az sql db restore --resource-group --server --name \ + --dest-name -recovered --deleted-time "2026-09-02T09:12:41" +``` + +Geo-restore is a different command, and the backup is addressed by resource id: + +```bash +az sql db geo-backup list --resource-group --server --output table + +az sql db geo-backup restore --geo-backup-id \ + --dest-database --dest-server \ + --resource-group +``` + +Long-term retention is a third command, with backup ids listed per region: + +```bash +az sql db ltr-backup list --location --server --database \ + --output table + +az sql db ltr-backup restore --backup-id \ + --dest-database --dest-server --dest-resource-group +``` + +Pick by reach, not by habit. The rows are Learn's, and the wrong choice wastes a restore rather +than taking a slower route: + +| Restore | Where it can land | What has to be true already | +|---|---|---| +| Point-in-time | the same server only, no cross-server, cross-subscription or cross-region | inside the retention window; the source is a primary, not a geo-secondary | +| Deleted database | the same server only | the server still exists; deleting a server deletes its databases and their backups together | +| Geo-restore | any server in any region, same subscription | backup storage redundancy was already `Geo` or `GeoZone`; it is not retroactive | +| Long-term retention | any server, and it outlives the source server | an LTR policy existed before the point you want | + +So if the logical server itself was deleted, long-term retention is the only route left. Restoring +across the Hyperscale boundary, in either direction, is not supported at all. + +## 4. Retention is a setting, and the default is seven days + +Both windows are set per database: + +```bash +az sql db str-policy set --resource-group --server --name \ + --retention-days 35 --diffbackup-hours 12 + +az sql db ltr-policy set --resource-group --server --name \ + --weekly-retention P4W --monthly-retention P12M --yearly-retention P10Y --week-of-year 1 +``` + +Short-term retention defaults to 7 days and takes 1 to 35, except DTU Basic which takes 1 to 7. +`--diffbackup-hours` accepts only 12 or 24. Each long-term retention value takes a minimum of 7 +days and a maximum of 10 years, and `--week-of-year` is 1 to 52. Those bounds are in both +the CLI's own help and the Learn pages below. + +## 5. The swap nobody automates + +The restore leaves two databases: the broken one and the new one holding the data. Learn names the +rename as the way to finish it. In T-SQL, measured on the engine above: + +```sql +ALTER DATABASE [app-db] MODIFY NAME = [app-db-broken]; +ALTER DATABASE [app-db-restored] MODIFY NAME = [app-db]; +``` + +Or the same swap from the control plane, which still works when nothing can connect to run T-SQL: + +```bash +az sql db rename --resource-group --server --name \ + --new-name -broken +az sql db rename --resource-group --server --name -restored \ + --new-name +``` + +Either way it is a cutover with a brief gap: pooled connections against the old name fail or +reconnect depending on the driver. The alternative is to repoint the connection string and rename +nothing, which has no gap but means finding every hardcoded copy of the database name, migration +tooling included. + +## 6. Getting recovered data out of the subscription + +Point-in-time restore is same-server and geo-restore is same-subscription, so neither reaches +another subscription or a workstation. Exporting the restored database is the only route out: + +```bash +sqlpackage /Action:Export /TargetFile:recovered.bacpac \ + /SourceServerName:.database.windows.net /SourceDatabaseName:-restored \ + /SourceUser: /SourcePassword:"$SQLPACKAGE_PASSWORD" /p:VerifyExtraction=true +``` + +A bacpac is a logical copy of one moment, not a restore point. It cannot recover anything the +retention window has already dropped; it only moves what a restore already produced. + +## Check it worked + +A restore command returning is not a recovered database. + +The database exists and is serving: + +```bash +az sql db show --resource-group --server --name -restored \ + --query "{name:name, status:status, sku:sku.name, tags:tags}" --output table +``` + +Expect `status` `Online`. Expect empty tags even on a healthy restore: Learn states a restore does +not carry the source's tags, so their absence is not a failure. + +The rows are the rows you wanted, which the control plane cannot tell you. Run this against the +restored name and again against the name the application uses, after the swap in section 5: + +```bash +sqlcmd -S .database.windows.net -d -restored -U \ + -P "$SQLCMD_PASSWORD" -Q "SELECT DB_NAME() AS connected_to, + SUM(p.rows) AS row_count, COUNT(DISTINCT t.object_id) AS table_count +FROM sys.tables AS t JOIN sys.partitions AS p + ON p.object_id = t.object_id AND p.index_id IN (0, 1);" +``` + +Equal counts across the two names, or a stated reason they differ, is the check. A restore to a +point before a bad backfill is expected to differ, and you should be able to say by how much. Until +the second run returns the restored data, the recovery is not finished. + +## Do not + +- Do not write `RESTORE DATABASE`, `BACKUP DATABASE`, any `RESTORE ... ONLY` variant, or + `SET RECOVERY` as the recovery step. All of them are refused, measured. +- Do not say a restore updates the existing database. Learn says outright that you can't overwrite + an existing database during restore. +- Do not stop when the restored database exists. The rename or connection string swap is part of + the recovery. +- Do not promise a geo-restore carries firewall rules, the Microsoft Entra administrator, or server + level logins to a different logical server. Those are server scoped, contained users are not, and + none of it was verified here. +- Do not quote a fixed geo-restore RPO or RTO. Learn's figures for this service are "typically + minutes or hours", dependent on storage replication and backup size; a fixed pair of hours comes + from a different product's page. +- Do not size the target casually. Learn notes a restore may need S3 or above, which you can scale + back down once it finishes. + +## References + +- [references/backup-restore-tsql-errors.md](references/backup-restore-tsql-errors.md): open before + restating any error number here as measured, or when an engine answers differently from the + table in section 1. +- [Recover using automated database backups](https://learn.microsoft.com/azure/azure-sql/database/recovery-using-backups): + read it before a restore that crosses a server, a subscription or a region; it is where the scope + limits in section 3 come from. +- [Automated backups](https://learn.microsoft.com/azure/azure-sql/database/automated-backups-overview): + read it when you need retention defaults or backup storage redundancy options. +- [Long-term retention](https://learn.microsoft.com/azure/azure-sql/database/long-term-retention-overview): + read it before promising a restore point older than the short-term window. +- [Business continuity overview](https://learn.microsoft.com/azure/azure-sql/database/business-continuity-high-availability-disaster-recover-hadr-overview): + read it when the requirement is near zero downtime, which failover groups and active + geo-replication solve and restore does not. +- `provision-azure-sql-db`: creating a database and its firewall rule in the first place. +- `provision-hyperscale`: Hyperscale restore mechanics, which differ from the general case here. +- `schema-migrations-safely`: rolling back a bad schema change without touching data. +- `sqlpackage-import-export`: the full bacpac and dacpac surface behind the one command above. diff --git a/plugins/microsoft-sql-ssms/skills/restore-and-recover/references/backup-restore-tsql-errors.md b/plugins/microsoft-sql-ssms/skills/restore-and-recover/references/backup-restore-tsql-errors.md new file mode 100644 index 0000000..97cf0dc --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/restore-and-recover/references/backup-restore-tsql-errors.md @@ -0,0 +1,75 @@ +# Backup and restore T-SQL on Azure SQL Database: every statement and its error + +## Contents + +- [What this file is](#what-this-file-is) +- [Engine identity](#engine-identity) +- [Statements refused outright](#statements-refused-outright) +- [The database rename swap, measured](#the-database-rename-swap-measured) +- [Backup system views that exist and return nothing locally](#backup-system-views-that-exist-and-return-nothing-locally) +- [What this file does NOT cover](#what-this-file-does-not-cover) + +## What this file is + +Every statement below was run against a live container engine on 2026-08-29 and the +output is copied verbatim, trimmed only of `sqlcmd` padding whitespace. Two rows were +re-measured afterwards and corrected, both toward a refusal: `sp_get_database_backup_policy` +on 2026-09-02 and `SET RECOVERY` on 2026-09-03; each says so in place. This is the evidence +behind the correction in SKILL.md. + +## Engine identity + +```sql +SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('Edition'), SERVERPROPERTY('ProductVersion'); +-- 5 SQL Azure 12.0.2000.8 +``` + +## Statements refused outright + +Each ran against a real database with a table and rows in it, not an empty one. + +| Statement | Result | +|---|---| +| `BACKUP DATABASE db TO DISK = '...'` | `Msg 40510, Level 16, State 1`: "Statement 'BACKUP DATABASE' is not supported in this version of SQL Server." | +| `BACKUP DATABASE db TO URL = 'https://...'` | Same `Msg 40510`, same text. The destination does not change the outcome; the statement is rejected before it looks at where the file would go. | +| `BACKUP LOG db TO DISK = '...'` | `Msg 40510`: "Statement 'BACKUP LOG' is not supported in this version of SQL Server." | +| `RESTORE DATABASE db FROM DISK = '...' WITH REPLACE` | `Msg 40510`: "Statement 'RESTORE DATABASE' is not supported in this version of SQL Server." | +| `RESTORE HEADERONLY FROM DISK = '...'` | `Msg 40510`: "Statement 'RESTORE VOLUME' is not supported in this version of SQL Server." Note the message names `RESTORE VOLUME`, not `RESTORE HEADERONLY`; the parser collapses the `RESTORE ... ONLY` family into one internal name before it reports the rejection. Match on the number, never the text. | +| `RESTORE FILELISTONLY FROM DISK = '...'` | Same `Msg 40510`, "RESTORE VOLUME" text. | +| `RESTORE VERIFYONLY FROM DISK = '...'` | Same `Msg 40510`, "RESTORE VOLUME" text. | +| `EXEC sys.sp_get_database_backup_policy 'db'` | `Msg 2812`: "Could not find stored procedure 'sys.sp_get_database_backup_policy'." Re-measured 2026-09-02 on Azure SQL Database (EngineEdition 5, `SQL Azure`, 12.0.2000.8, S0) and on the container: both answer 2812, and `sys.all_objects` holds nothing matching `%backup_policy%`. This row previously said `Msg 15817`, a real message ("The stored procedure is not available in this version of SQL Server") but the wrong one: 15817 is what a version-gated procedure that EXISTS returns, and would have told a reader this one was merely gated off. | +| `ALTER DATABASE db SET RECOVERY FULL` or `SIMPLE` | `Msg 40517, Level 16, State 1`: "Keyword or statement option 'RECOVERY' is not supported in this version of SQL Server." Not `40510`: that refuses a whole statement, while `40517` refuses one option of a statement that is otherwise supported, and `ALTER DATABASE ... MODIFY NAME` below still works. Learn documents `40517` with this exact text, and lists recovery models among the syntax Azure SQL Database does not support. Until 2026-09-03 this row said the statement RAN and moved `recovery_model_desc`, making it the one accepted statement in the family; re-measured, it is refused like the rest. | +| `CREATE DATABASE copy_name AS COPY OF db` | `Msg 102, Level 15`: "Incorrect syntax near 'COPY'." A **parse** error, not a `40510` refusal, so the container's parser does not recognise the clause at all. Azure SQL Database does accept `CREATE DATABASE ... AS COPY OF` for an online database copy, a different operation from restore. Not cross-checked against a live logical server; read Microsoft Learn before writing it. | + +## The database rename swap, measured + +```sql +ALTER DATABASE dq_restore2 MODIFY NAME = dq_restore_swapped; +-- The database name 'dq_restore_swapped' has been set. + +SELECT name FROM sys.databases WHERE name LIKE 'dq_restore%'; +-- dq_restore_swapped +``` + +`ALTER DATABASE ... MODIFY NAME` is ordinary T-SQL, works on this engine, and is the +mechanism for putting a restored database's name back where the application expects it. +It renames in place; it does not merge or delete anything, so the original database +still has to be renamed out of the way or dropped separately. + +## Backup system views that exist and return nothing locally + +`sys.dm_database_backups`, `sys.dm_database_backup_lineage` and +`sys.fn_db_backup_file_snapshots(NULL)` all resolved as real objects and returned zero +rows, rather than an invalid-object error. The container does not run the Azure backup +service, so there is nothing for them to report.The singular name +`sys.fn_db_backup_file_snapshot` does not exist: `Msg 208`, invalid object name. The real +object is plural. + +## What this file does NOT cover + +Point-in-time restore, geo-restore, restore from long-term retention, and the +`az sql db restore` / `Restore-AzSqlDatabase` / REST surface all require a real logical +server and were not exercised here. Nothing in this file is evidence about how long a +restore takes, what a restored database is named by default, or whether firewall rules +or the Microsoft Entra administrator carry over. SKILL.md links Microsoft Learn at each +of those points instead of guessing. diff --git a/plugins/microsoft-sql-ssms/skills/restore-and-recover/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/restore-and-recover/skill.spec.jsonc new file mode 100644 index 0000000..440e332 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/restore-and-recover/skill.spec.jsonc @@ -0,0 +1,138 @@ +// skills/restore-and-recover/skill.spec.jsonc +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "restore-and-recover", + "domain": "operate-and-recover", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked to recover an Azure SQL Database, an agent trained on the boxed engine reaches for RESTORE DATABASE ... WITH REPLACE, BACKUP DATABASE, RESTORE HEADERONLY or ALTER DATABASE ... SET RECOVERY, and none of that surface exists on the service. Measured 2026-08-29 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure': BACKUP DATABASE, BACKUP LOG, RESTORE DATABASE, RESTORE HEADERONLY, RESTORE FILELISTONLY and RESTORE VERIFYONLY are each refused with Msg 40510, sys.sp_get_database_backup_policy does not exist at all, failing with Msg 2812, and ALTER DATABASE ... SET RECOVERY is refused with Msg 40517, so the family has no exception. Microsoft Learn says it plainly: you recover using the portal, PowerShell, the Azure CLI or the REST API, and you can't use Transact-SQL. The second wrong belief compounds the first. Learn again: you can't overwrite an existing database during restore. Every restore, point-in-time, geo or long-term, creates a new database under the name given at restore time, and moving that database to where the application connects, by rename or by a connection string change, is a step the developer performs. Nothing does it automatically and nothing warns when it is skipped, so the failure surfaces later as a connection error blamed on the application. The third belief the skill corrects is about reach: point-in-time restore is same-server only, geo-restore is same-subscription only and needs Geo or GeoZone backup storage configured beforehand, and deleting the logical server takes its databases and their point-in-time backups with it, leaving long-term retention as the only surviving route.", + "value_probe": { + "gate": "The answer recovers to a new database from the service's own automatic backups using a control plane tool, and names the explicit cutover that puts the application onto it, without proposing a T-SQL restore or an overwrite of the live database.", + "task": "At about 09:40 this morning a deploy script ran against the wrong environment and emptied four tables in our production Azure SQL Database. The application is still up and still taking writes into other tables. We want the state as of 09:35 back, and we want the application reading the recovered data by end of day.\n\nOur runbook says to take the most recent backup and put it back over the top of the live database. One of the team also wants to switch the database into a mode that keeps more log around so that this is easier to deal with next time, and someone else has suggested we recover onto a different server in another subscription so it stays isolated while we check it.\n\nI have the portal, the Azure CLI and a SQL client open in front of me. Walk me through exactly what to do, in order.", + "states": [ + "Recovery on this service is driven from the portal, PowerShell, the Azure CLI or the REST API, and not from T-SQL at all.", + "Every restore produces a new database under the name given at restore time, and nothing overwrites the existing one.", + "Putting the application onto the recovered data is a manual step, by rename or by changing the connection string, that nothing performs and nothing warns about when it is skipped.", + "The recovery model option is refused on this service, while the ALTER DATABASE statement itself remains valid for other options.", + "A point in time restore lands on the same logical server, so recovering onto a different server or subscription is a different route with its own prerequisites." + ], + "refutes": [ + "RESTORE DATABASE, or some other T-SQL restore statement, is the way to recover this database.", + "A restore can overwrite the live database in place, so the application keeps working with no further step once it finishes.", + "The recovery model of the database can be changed to retain more log for easier recovery next time.", + "A point in time restore can be directed at a different logical server or a different subscription.", + "ALTER DATABASE is rejected wholesale on this service, so no option of that statement is usable." + ] + }, + "posture": ["read", "provision", "admin"], + "maturity": "preview", + "applies_to": ["azure-sql-db"], + "related": [ + "provision-azure-sql-db", + "provision-hyperscale", + "schema-migrations-safely", + "sqlpackage-import-export", + "diagnose-connection-errors", + "entra-id-auth" + ], + "validation": { + "target": "container", + "setup": [ + "a database with at least one table and rows in it", + "administrative access sufficient to run ALTER DATABASE and CREATE DATABASE" + ], + "assert": [ + "BACKUP DATABASE, BACKUP LOG, RESTORE DATABASE, RESTORE HEADERONLY, RESTORE FILELISTONLY and RESTORE VERIFYONLY are all refused with Msg 40510, and ALTER DATABASE ... SET RECOVERY with Msg 40517", + "no answer proposes RESTORE or BACKUP T-SQL as the recovery mechanism", + "every answer states that a restore creates a new database rather than overwriting the source in place", + "an answer describing a completed recovery includes the rename or connection string swap back to the name the application uses", + "an answer that proposes a restore across servers, subscriptions or regions names which of the three restore types can actually reach there", + "no answer states a retention window, RPO number or CLI flag as fact without attributing it to Microsoft Learn or to the CLI's own help" + ], + "probes": [ + { + "sql": "DECLARE @stmt nvarchar(max) =\n N'BACKUP DATABASE ' + QUOTENAME(DB_NAME()) + N' TO DISK = ''/var/opt/mssql/data/probe-restore.bak'';';\nEXEC sys.sp_executesql @stmt;", + "expect": { + "error": 40510 + }, + "why": "BACKUP DATABASE is the first thing a SQL Server habit reaches for, and it is refused outright rather than failing on the destination. Measured 2026-08-29 against an engine reporting EngineEdition 5: Msg 40510, statement not supported in this version of SQL Server. The database name is bound from DB_NAME() so the refusal cannot be confused with an unresolved name." + }, + { + "sql": "DECLARE @stmt nvarchar(max) =\n N'BACKUP LOG ' + QUOTENAME(DB_NAME()) + N' TO DISK = ''/var/opt/mssql/data/probe-restore.trn'';';\nEXEC sys.sp_executesql @stmt;", + "expect": { + "error": 40510 + }, + "why": "BACKUP LOG is refused the same way as BACKUP DATABASE, which matters because a log backup is what someone reaches for second, after being told full backups are handled by the service. Measured 2026-08-29, Msg 40510." + }, + { + "sql": "RESTORE DATABASE probe_restore_target FROM DISK = '/var/opt/mssql/data/probe-restore.bak' WITH REPLACE;", + "expect": { + "error": 40510 + }, + "why": "The headline claim of the skill is that there is no restore T-SQL on this service. Measured 2026-08-29, Msg 40510. WITH REPLACE is carried deliberately because that is the clause an agent adds when it believes a restore overwrites the database in place, which is the second wrong belief this skill corrects." + }, + { + "sql": "RESTORE HEADERONLY FROM DISK = '/var/opt/mssql/data/probe-restore.bak';", + "expect": { + "error": 40510 + }, + "why": "The whole RESTORE ... ONLY family is refused, and the message names RESTORE VOLUME rather than RESTORE HEADERONLY because the parser collapses the family into one internal name before rejecting it. Measured 2026-08-29. Matching on the number rather than the text is the only stable way to probe this." + }, + { + "sql": "DECLARE @db sysname = DB_NAME();\nEXEC sys.sp_get_database_backup_policy @db;", + "expect": { + "error": 2812 + }, + "why": "The procedure an agent reaches for once it accepts that backups are automatic: Msg 2812, could not find stored procedure. The procedure does not exist on the service. Recorded as Msg 15817 on 2026-08-29 and corrected on 2026-09-02 after the live lane disagreed with it: measured against Azure SQL Database (EngineEdition 5, SQL Azure, 12.0.2000.8, S0) AND the container, both answer 2812, and sys.all_objects holds nothing matching %backup_policy%. The skill's claim is unchanged and still true: an answer that reads the backup policy from T-SQL cannot be built at all." + }, + { + "sql": "DECLARE @stmt nvarchar(max) =\n N'ALTER DATABASE ' + QUOTENAME(DB_NAME()) + N' SET RECOVERY FULL;';\nEXEC sys.sp_executesql @stmt;", + "expect": { + "error": 40517 + }, + "why": "The last habit in the family, and the second number in this skill the engine disproved. Recorded 2026-08-29 as the one statement ACCEPTED, moving sys.databases.recovery_model_desc and therefore the trap the skill warned about; re-measured 2026-09-03 and refused: Msg 40517, keyword or statement option 'RECOVERY' is not supported. Microsoft Learn documents 40517 with that exact text and lists recovery models among the unsupported syntax. Stronger than the claim it replaces: the family now has no exception for a reader to trip over." + }, + { + "cmd": "az sql db restore --help 2>/dev/null | grep -q -- '--deleted-time' && az sql db geo-backup restore --help 2>/dev/null | grep -q -- '--geo-backup-id' && az sql db ltr-backup restore --help 2>/dev/null | grep -q -- '--backup-id'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The three restore command lines in section 3 are only useful if those flags exist in the CLI the reader has. Taken from az 2.90.0's own help on 2026-09-03 and asserted here so a CLI rename is caught by the lane rather than by a developer mid-incident. Runs on the host and needs no subscription or network: --help is answered locally. `host` because the Azure CLI is a client tool a developer installs and is not in the engine image." + }, + { + "cmd": "sqlpackage /Action:Export /help:True 2>/dev/null | grep -q '^/TargetFile:'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "Section 6 sends the recovered data out of the subscription with a SqlPackage Export, and /TargetFile is the parameter that names the bacpac. Verified against SqlPackage 170.4.83.3 on 2026-09-03. `where` is host and not container: SqlPackage is a client tool a developer installs, it is not in the engine image, and five probes shipped on 2026-09-02 without this field failed with 'sqlpackage: command not found' for exactly that reason." + } + ] + }, + "triggering": { + "explicit": [ + "How do I restore my Azure SQL database to before this bad deployment", + "Write the RESTORE DATABASE command to recover my Azure SQL database", + "What is the point-in-time restore command for Azure SQL Database", + "How do I geo-restore my database after a region outage", + "How do I recover a database using long-term retention", + "Give me the BACKUP DATABASE syntax for Azure SQL Database" + ], + "implicit": [ + "someone just dropped the orders table in production, how do I get it back", + "we ran a bad backfill an hour ago and need last night's data", + "the primary region is down, how do I bring the database back up somewhere else", + "I ran BACKUP DATABASE against my Azure SQL database and got an error I don't understand", + "the restored database has a different name than my app expects and now nothing connects", + "how far back can I recover this database" + ], + "negative": [ + "Is this migration safe to run against production while the app is live", + "Export my database to a bacpac file", + "Create the Azure SQL database and give me a connection string", + "Should I use Hyperscale for this workload", + "Set up active geo-replication for read scale-out", + "My app connects fine locally but gets login failed once it is deployed to Azure" + ] + } +} diff --git a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant deleted file mode 120000 index f1b3cb7..0000000 --- a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant +++ /dev/null @@ -1 +0,0 @@ -../../../skills/rls-multi-tenant \ No newline at end of file diff --git a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/SKILL.md b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/SKILL.md new file mode 100644 index 0000000..1350449 --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/SKILL.md @@ -0,0 +1,391 @@ +--- +name: rls-multi-tenant +description: >- + Builds tenant isolation on Azure SQL Database that a test can prove, with a row level security + policy whose filter predicate and block predicate are written together, because a filter alone + still accepts a cross-tenant write and hides the row from the app that made it. Use when asked + to add row level security, isolate tenants in a shared table, write a security policy or + predicate function, set the current tenant through SESSION_CONTEXT or a database user per + tenant, or prove one tenant cannot read another; and when a multi-tenant app returns the wrong + tenant's rows under load, retrieval returns another tenant's chunk, a policy is in place and + all rows are still visible, or error 33504 appears on an insert or update. Covers pooling + against a session-scoped tenant id, who can turn a policy off, and the isolation test. Identity + is entra-id-auth, the table design design-azure-sql-schema. +--- + +# Tenant isolation that holds, and the test that proves it + +**The deliverable is the test, not the policy.** The value is an assertion that goes red when +isolation breaks, because every way it breaks here is silent: no error, no warning, no log line, +and a catalog view that still reports the policy enabled. + +Measured 2026-08-28 against a live engine reporting `EngineEdition` `5` and `Edition` `SQL Azure`. +Open [the measured runs](references/block-predicates-pooling-and-bypass.md) when you need the +statement behind a count or a message number below, or before changing one. + +## 1. A filter predicate is not the boundary + +Microsoft Learn states it plainly: the application can `INSERT` rows, even if they will be filtered +during any other operation. Measured as a low-privilege user holding tenant id 1, filter predicate +only: + +| Statement | Result | What it cost | +|---|---|---| +| `INSERT` a row stamped `tenant_id = 2` | **succeeded** | the row exists, invisible to its author, visible to tenant 2 | +| `UPDATE` one of my own rows to `tenant_id = 2` | **succeeded** | my row moved into another tenant, silently | +| `UPDATE` a row of tenant 2 | 0 rows affected | the filter protects the read behind a write | +| `DELETE` a row of tenant 2 | 0 rows affected | same | +| `INSERT` a key that already exists in tenant 2 | `Msg 2627` | the key space of another tenant is probeable | + +The first two rows are the leak. The application that wrote them can never read them back and never +delete them, because the filter now hides them from their own author. Recovery meant disabling the +policy. + +The last row is separate and smaller. A filter predicate hides rows, not their existence: `Msg 2627` +on a duplicate key and success on a free one let a caller enumerate another tenant's keys one +attempt at a time. Use identifiers with no guessable ordering. + +## 2. The block predicate, and why it is two predicates + +Learn defines four block operations: `AFTER INSERT`, `AFTER UPDATE`, `BEFORE UPDATE` and +`BEFORE DELETE`. Adding the first two turned both cross-tenant writes into: + +```text +Msg 33504: The attempted operation failed because the target object '..
' +has a block predicate that conflicts with this operation. +``` + +The legitimate write into the caller's own tenant still succeeded. + +**`AFTER UPDATE` is the one people leave out.** Measured: with the filter predicate and +`AFTER INSERT` in place but no `AFTER UPDATE`, the cross-tenant insert was refused and the +tenant-move update **still succeeded**. One row left the tenant with no error. + +```sql +CREATE FUNCTION sec.fn_tenant(@tenant_id int) +RETURNS TABLE +WITH SCHEMABINDING +AS +RETURN SELECT 1 AS ok + WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int); +GO + +CREATE SECURITY POLICY sec.p_orders + ADD FILTER PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders, + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders AFTER INSERT, + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders AFTER UPDATE + WITH (STATE = ON); +GO + +-- the other half of the guarantee: the application never rewrites a tenant id +DENY UPDATE ON dbo.orders(tenant_id) TO app_user; +``` + +Learn adds two things. The optimizer skips an `AFTER UPDATE` block predicate when the update changed +no column the predicate reads, so ordinary updates pay nothing for it. And Learn's middle-tier +example omits `AFTER UPDATE` only because it denies `UPDATE` on the tenant column instead. Take +both. `BEFORE UPDATE` and `BEFORE DELETE` are redundant while a filter predicate is present, since +the filter has already removed the other tenant's rows from the statement. + +## 3. How the tenant reaches the predicate, and what pooling does to it + +Two designs, and the difference is measurable rather than stylistic. + +### `SESSION_CONTEXT` is a variable, not an identity + +Learn is explicit that any user can set a session context for their session, and the measurement +matches: a user granted only `CONNECT` plus `SELECT`, `INSERT`, `UPDATE` and `DELETE` on one table +set its own tenant id to 2 and read tenant 2's rows. No grant was needed, and none can be revoked +to stop it. + +So a `SESSION_CONTEXT` design is exactly as strong as the guarantee that nothing +attacker-influenced ever reaches that connection as SQL. `prevent-sql-injection` owns that: +open it whenever any part of a statement here is assembled from input instead of passed as a +parameter, because under this design a successful injection rewrites the tenant boundary itself. + +### The failure that reaches production + +The tenant id outlives the request, because the connection does. Measured over a five connection +pool, 200 concurrent requests alternating between two tenants: + +| Application shape | Cross-tenant reads in 200 requests | +|---|---| +| Set the context in one pooled request, run the query in another | **103**, then **91** on a repeat | +| Set the context and run the query on one acquired connection | 0 | +| A database user per tenant, one pool per tenant, no session context | 0 | + +Half the requests returned another tenant's rows and **no caller saw an error**. The application set +the tenant on every request; it handed the set and the query to the pool separately, and the pool +gave them different connections. Two rules follow: + +1. **Set the tenant and read on the same acquired connection**, in one batch, one transaction, or + one explicitly held connection. Not two calls to a pool. +2. **Never leave the tenant set when a connection returns to the pool.** That failure is open, not + closed: the next caller that forgets reads the previous tenant's rows. + +`@read_only = 1` looks like the fix, and here the documentation and the measurement point different +ways. Learn presents it as preventing the value changing again until the connection returns to the +pool, which is exactly what it does. Measured on a reused pooled connection, the next tenant's +**correct** attempt to claim its own id was refused with `Msg 15664` while the request that forgot +to set anything still read the previous rows. It is safe only where the connection ends with the +request. + +### A database user per tenant + +The predicate reads `USER_NAME()` or `DATABASE_PRINCIPAL_ID()` instead of session state, resolved +through a mapping table: + +```sql +CREATE FUNCTION sec.fn_tenant_by_user(@tenant_id int) +RETURNS TABLE +WITH SCHEMABINDING +AS +RETURN SELECT 1 AS ok FROM sec.tenant_map m + WHERE m.db_user = USER_NAME() AND m.tenant_id = @tenant_id; +``` + +Measured: setting `SESSION_CONTEXT` under this predicate changed nothing, and borrowing another +tenant's identity with `EXECUTE AS USER` was refused with `Msg 15517`, since impersonation is a +permission the tenant does not hold. Connection pools are keyed by connection string, so each +tenant gets its own pool and there is no shared connection to leak through. + +The cost is one login, one database user, one connection string and one pool per tenant, which fits +hundreds of tenants and not tens of thousands. Choose it when isolation has to survive an +application bug rather than depend on its absence. + +## 4. Who bypasses the policy + +| Principal | Reads through the policy | Can turn it off | +|---|---|---| +| A plain application user | no, 0 rows with no tenant set | no, `Msg 33268` | +| The table owner, which is also the database owner | **no**, 0 rows with no tenant set | yes | +| A member of `db_owner` | **no**, 0 rows with no tenant set | yes, in one statement, then reads every row | +| `ALTER ANY SECURITY POLICY` alone | no | **no**, `Msg 33268` | +| `ALTER` on the schema holding the policy alone | no | **no**, `Msg 33268` | +| Both of the above together | no | yes, in one statement, then reads every row | + +**Ownership is not a read-through.** Learn says a `dbo` user, a `db_owner` member and the table +owner are all filtered or blocked as the policy defines, and all three returned zero rows with no +tenant set. What they hold is the ability to disable, a separate and auditable statement. + +**`ALTER ANY SECURITY POLICY` alone is not enough.** Learn splits the requirement, altering a policy +against that permission and creating or dropping one against `ALTER` on the schema. Measured, +disabling one needed both: either alone failed with `Msg 33268`, reported as a missing object rather +than a permission failure. Grant them as a pair, to a principal no application connects as. + +## 5. A policy can be present and not enforcing, in three ways + +All three look healthy from the application: rows come back, nothing errors. + +| Shape | What the application sees | What gives it away | +|---|---|---| +| `WITH (STATE = OFF)` | every tenant's rows | `sys.security_policies.is_enabled = 0` | +| A predicate that returns a row when the tenant is unset | every tenant's rows, to a session that set nothing | nothing in any catalog view; `is_enabled` is still 1 | +| A second tenant-scoped table nobody added to the policy | that table unfiltered, while the covered table is correct | no predicate rows for that object | + +A predicate of the form `WHERE @tenant_id = OR IS NULL` returned all four rows +to a session with no tenant set, while every catalog view reported a healthy enabled policy. **Predicates fail closed or they are not predicates.** Never add +an escape clause for administrative sessions. + +The third is a sweep, not a review. Run it after every migration that adds a table: + +```sql +SELECT QUOTENAME(SCHEMA_NAME(t.schema_id)) + '.' + QUOTENAME(t.name) AS table_name, + SUM(CASE WHEN sp.predicate_type = 0 THEN 1 ELSE 0 END) AS filter_predicates, + SUM(CASE WHEN sp.predicate_type = 1 THEN 1 ELSE 0 END) AS block_predicates, + MIN(CAST(pol.is_enabled AS int)) AS policy_enabled +FROM sys.tables t +JOIN sys.columns c + ON c.object_id = t.object_id AND c.name = N'tenant_id' +LEFT JOIN sys.security_predicates sp ON sp.target_object_id = t.object_id +LEFT JOIN sys.security_policies pol ON pol.object_id = sp.object_id +GROUP BY t.schema_id, t.name +HAVING SUM(CASE WHEN sp.predicate_type = 1 THEN 1 ELSE 0 END) < 2 + OR MIN(CAST(pol.is_enabled AS int)) IS NULL + OR MIN(CAST(pol.is_enabled AS int)) = 0; +``` + +`predicate_type` is `0` for filter and `1` for block, read back from `sys.security_predicates` +rather than recalled. Read the list rather than count it: a tenant mapping table carries a +`tenant_id` column, legitimately has no predicate, and is dismissed once. Anything else in it is a +tenant table that is not isolated. Verified by planting a failure: drop one block predicate from a +correct table and that table appears. + +One shape the sweep misses, documented by Learn rather than measured here: predicates are not +replicated to a system-versioned table's history table, which needs its own predicate by name. + +## 6. The local rehearsal gap for retrieval + +A tenant predicate on a chunk table is the cheapest defence against a retrieval answering one +tenant with another tenant's text, and **on the Azure SQL Database container it cannot be rehearsed +on an indexed table**. Reproduced on 300 rows with a real vector index, in both orders: index first +then `CREATE SECURITY POLICY` gives `Msg 37579`, policy first then `CREATE VECTOR INDEX` gives +`Msg 42244`. On Azure SQL Database the two coexist. So the isolation control is exactly the part of a +retrieval design that never runs locally beside the index it will meet in production. + +What does work locally: the filter predicate applies correctly to exact search over +`VECTOR_DISTANCE` with no vector index, and two tenants over 300 chunks each got back only their own +rows. Rehearse the predicate unindexed, keep index creation out of the local path, and run the +isolation test again once the index exists on the cloud database. `rag-local-with-container` +measured the same pair. + +## Check it worked + +Three checks, cheapest first, against the reference fixture: `dbo.orders` holding four rows, two +each for tenants 1 and 2, and `app_user` granted only `CONNECT` and the four table permissions. + +**One: the filtered count is smaller than the table, and the administrator is not exempt.** Run it +on an administrative connection, which is the only one that can impersonate: + +```sql +EXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1; +SELECT COUNT(*) AS admin_tenant_1 FROM dbo.orders; +EXECUTE AS USER = 'app_user'; +SELECT COUNT(*) AS app_tenant_1 FROM dbo.orders; +EXEC sys.sp_set_session_context @key = N'tenant_id', @value = 2; +SELECT COUNT(*) AS app_tenant_2 FROM dbo.orders; +EXEC sys.sp_set_session_context @key = N'tenant_id', @value = NULL; +SELECT COUNT(*) AS app_no_tenant FROM dbo.orders; +REVERT; +``` + +Expected against four seeded rows: `2`, `2`, `2`, `0`, and the two twos are different rows. A `4` +anywhere means the policy is not enforcing, which is section 5; zero everywhere means the predicate +is not reading the key you set. + +**Two: a cross-tenant write raises rather than vanishing.** + +```sql +EXECUTE AS USER = 'app_user'; +EXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1; +INSERT INTO dbo.orders (order_id, tenant_id, customer, amount) VALUES (901, 2, N'probe', 1.00); +REVERT; +``` + +Expect `Msg 33504`. If it succeeds you have the leak in section 1, and the row is already invisible +to the connection that wrote it. + +**Three: the isolation test, as the application's own user, on every deployment**, saved as +`tenant-isolation.sql`: + +```sql +SET NOCOUNT ON; +DECLARE @mine int = 1, @theirs int = 2, @fail int = 0, @n int; +-- unique per run: a failing run strands a probe row it can no longer see or delete +DECLARE @probe int = 900000 + ABS(CHECKSUM(NEWID())) % 90000; + +EXEC sp_set_session_context @key = N'tenant_id', @value = 1; + +-- 1. every visible row is mine +SELECT @n = COUNT(*) FROM dbo.orders WHERE tenant_id <> @mine; +IF @n <> 0 BEGIN PRINT 'FAIL 1: the filter predicate leaks another tenant''s rows'; SET @fail = 1; END + +-- 2. an unfiltered aggregate cannot see more than the filtered set +SELECT @n = COUNT(*) FROM dbo.orders; +IF @n <> (SELECT COUNT(*) FROM dbo.orders WHERE tenant_id = @mine) + BEGIN PRINT 'FAIL 2: aggregate disagrees with the filtered set'; SET @fail = 1; END + +-- 3. a write addressed to another tenant is REFUSED, not silently accepted +BEGIN TRY + INSERT INTO dbo.orders (order_id, tenant_id, customer, amount) + VALUES (@probe, @theirs, N'probe', 1.00); + PRINT 'FAIL 3: cross-tenant INSERT accepted, no BLOCK PREDICATE AFTER INSERT'; + SET @fail = 1; +END TRY +BEGIN CATCH + IF ERROR_NUMBER() <> 33504 BEGIN PRINT CONCAT('FAIL 3: unexpected error ', ERROR_NUMBER()); SET @fail = 1; END +END CATCH + +-- 4. moving one of my own rows into another tenant is REFUSED +INSERT INTO dbo.orders (order_id, tenant_id, customer, amount) VALUES (@probe + 1, @mine, N'probe', 1.00); +BEGIN TRY + UPDATE dbo.orders SET tenant_id = @theirs WHERE order_id = @probe + 1; + PRINT 'FAIL 4: tenant-move UPDATE accepted, no BLOCK PREDICATE AFTER UPDATE'; + SET @fail = 1; +END TRY +BEGIN CATCH + IF ERROR_NUMBER() <> 33504 BEGIN PRINT CONCAT('FAIL 4: unexpected error ', ERROR_NUMBER()); SET @fail = 1; END +END CATCH +DELETE FROM dbo.orders WHERE order_id = @probe + 1; + +-- 5. no tenant set means no rows: the predicate fails closed +EXEC sp_set_session_context @key = N'tenant_id', @value = NULL; +SELECT @n = COUNT(*) FROM dbo.orders; +IF @n <> 0 BEGIN PRINT 'FAIL 5: the predicate fails OPEN when no tenant is set'; SET @fail = 1; END + +IF @fail = 1 THROW 50001, 'tenant isolation test FAILED', 1; +PRINT 'tenant isolation: ALL PASS'; +``` + +```bash +export SQLCMDPASSWORD='' +sqlcmd -S .database.windows.net,1433 -d -U app_user -C \ + -b -m-1 -V 16 -i tenant-isolation.sql -o tenant-isolation.out +echo "exit $?" +grep -nE '^FAIL |^Msg ' tenant-isolation.out +``` + +Expect exit `0`, `tenant isolation: ALL PASS` in the file, and no output from `grep`. `THROW` sets +severity 16, above the severity 10 `-b` ignores, so a failed assertion does set a non-zero exit; +`-m-1` puts every message in the file, including the `PRINT` naming which assertion went red. + +**`-m-1` is an ODBC `sqlcmd` instruction**, the 18.x build from `mssql-tools18`. Measured +2026-09-05, go-sqlcmd 1.10.0 prints no `Msg` header on a severity 10 message at any `-m` value, so +there the `PRINT` lines arrive with no number. `build-app-on-azure-sql` tells the builds apart. + +**Each failure was planted and watched going red**, which is the only evidence an assertion asserts +anything: + +| Planted | What went red | +|---|---| +| Dropped the `AFTER INSERT` block predicate | 3 | +| Dropped the `AFTER UPDATE` block predicate | 4 | +| `WITH (STATE = OFF)` | 1, 2, 4, 5 | +| Swapped in the predicate that returns rows when the tenant is unset | 5 | + +Assertions 1 and 2 are the same claim from two angles: 2 catches a predicate that filters a direct +scan and not an aggregate. Add the pooling case in application test code, because no single +connection reproduces it: issue concurrent requests for two tenants against one pool and assert +every response carries only its own tenant, the shape that measured 103 wrong answers in 200. + +## Do not + +- Do not ship a filter predicate on its own. A cross-tenant insert succeeds under it and the row is + then unreadable and undeletable by the application that wrote it. +- Do not add `AFTER INSERT` and stop. The tenant-move update was measured succeeding with that block + predicate in place. +- Do not set the tenant in one pooled call and query in another, and do not reach for + `@read_only = 1` to make that safe. It refuses the correct caller and leaves the forgetful one + leaking. +- Do not treat `SESSION_CONTEXT` as an identity. Any caller on the connection rewrites it with no + permission, so one concatenated statement defeats the whole policy. +- Do not assume the table owner or `db_owner` reads through the policy. They do not. They disable + it, which is a different and visible act. +- Do not grant `ALTER ANY SECURITY POLICY` believing it is inert alone. Paired with `ALTER` on the + policy's schema it is a full read of every protected table, in one statement. +- Do not read `is_enabled = 1` as proof of isolation. A fail-open predicate and an uncovered table + both leave it at 1. +- Do not conclude row level security is unavailable because the container refuses it beside a vector + index. The refusal is about the index, not about the policy. +- Do not leave the tenant column nullable. A row that arrives with no tenant is invisible to every + tenant and belongs to none. + +## References + +- Open [the measured runs](references/block-predicates-pooling-and-bypass.md) when a number above + is disputed, before changing one, or to reproduce the pooling harness and the four planted + failures. +- Read [Row-level security](https://learn.microsoft.com/sql/relational-databases/security/row-level-security) + before designing a predicate more complex than one column, for the block operations, the + permission split, and the cross-feature list that names indexed views and temporal tables. +- Read [sp_set_session_context](https://learn.microsoft.com/sql/relational-databases/system-stored-procedures/sp-set-session-context-transact-sql) + before choosing session context over a user per tenant: the 8,000 byte value limit, the 1 MB + session total, and the `@read_only` argument. +- Read [sqlcmd utility](https://learn.microsoft.com/sql/tools/sqlcmd/sqlcmd-utility) when wiring + the isolation test into a pipeline, for what `-b`, `-m` and `-V` do. +- `prevent-sql-injection` when any statement here is assembled rather than parameterised, which is + what a session context design depends on. `entra-id-auth` for getting an identity onto a working + connection, and `least-privilege-database-roles` for the roles an application connects as. +- `design-azure-sql-schema` for the tenant column and its place in the key, `t-sql-correctness` for + the predicate function as T-SQL, and `rag-on-azure-sql` with `vector-search-azure-sql` for the + retrieval this policy sits under. diff --git a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/references/block-predicates-pooling-and-bypass.md b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/references/block-predicates-pooling-and-bypass.md new file mode 100644 index 0000000..208ef2b --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/references/block-predicates-pooling-and-bypass.md @@ -0,0 +1,412 @@ +# Block predicates, pooling and bypass: the runs behind tenant isolation + +## Contents + +- [How this was measured](#how-this-was-measured) +- [The fixture](#the-fixture) +- [Run 1: filter predicate only](#run-1-filter-predicate-only) +- [Run 2: the block predicate](#run-2-the-block-predicate) +- [Run 3: AFTER INSERT without AFTER UPDATE](#run-3-after-insert-without-after-update) +- [Run 4: the duplicate key probe](#run-4-the-duplicate-key-probe) +- [Run 5: session context under connection pooling](#run-5-session-context-under-connection-pooling) +- [Run 6: read only session context](#run-6-read-only-session-context) +- [Run 7: a database user per tenant](#run-7-a-database-user-per-tenant) +- [Run 8: who bypasses the policy](#run-8-who-bypasses-the-policy) +- [Run 9: present but not enforcing](#run-9-present-but-not-enforcing) +- [Run 10: the vector index conflict](#run-10-the-vector-index-conflict) +- [Run 11: the isolation test, and the four planted failures](#run-11-the-isolation-test-and-the-four-planted-failures) +- [Run 12: the published snippets, run verbatim](#run-12-the-published-snippets-run-verbatim) +- [What is documented rather than measured](#what-is-documented-rather-than-measured) +- [How to reproduce](#how-to-reproduce) + +## How this was measured + +One local Azure SQL Database container, started from the preview registry image, reporting: + +```text +EngineEdition Edition +5 SQL Azure +``` + +Every statement was run through the command line client inside the container. The concurrency +runs used a Node client with an explicit pool size against the same engine over a mapped port. + +Two client identities were used throughout: + +- `sa`, which maps to `dbo` in the user database. `IS_ROLEMEMBER('db_owner')` returned `1` and + `IS_SRVROLEMEMBER('sysadmin')` returned `0`, which is the Azure SQL Database shape. +- a login and matching database user granted only `CONNECT` plus `SELECT`, `INSERT`, `UPDATE` and + `DELETE` on one table. This is the application principal in every run below. + +A contained user was not used, because the container refuses one. Create the login on `master` +and the user in the application database. + +## The fixture + +```sql +CREATE SCHEMA sec; +GO +CREATE TABLE dbo.orders ( + order_id int NOT NULL PRIMARY KEY, + tenant_id int NOT NULL, + customer nvarchar(50) NOT NULL, + amount decimal(10,2) NOT NULL +); +INSERT INTO dbo.orders VALUES + (1, 1, N'acme', 100.00), + (2, 1, N'acme', 200.00), + (3, 2, N'globex', 300.00), + (4, 2, N'globex', 400.00); +GO +CREATE FUNCTION sec.fn_tenant(@tenant_id int) +RETURNS TABLE +WITH SCHEMABINDING +AS +RETURN SELECT 1 AS ok + WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int); +GO +CREATE SECURITY POLICY sec.p_orders + ADD FILTER PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders + WITH (STATE = ON); +``` + +`sys.security_policies` then reported `p_orders`, `is_enabled = 1`, `is_schema_bound = 1`. + +## Run 1: filter predicate only + +As the application user, with `sp_set_session_context @key = N'tenant_id', @value = 1`. + +| Statement | Server response | Rows the writer could then see | +|---|---|---| +| `SELECT order_id, tenant_id FROM dbo.orders` | 2 rows: 1, 2 | correct | +| `INSERT INTO dbo.orders VALUES (5, 2, N'globex', 500.00)` | **succeeded, no error** | 1, 2. Row 5 vanished | +| `UPDATE dbo.orders SET tenant_id = 2 WHERE order_id = 1` | **succeeded, 1 row** | 2 only. Row 1 vanished | +| `UPDATE dbo.orders SET amount = 999 WHERE order_id = 3` | 0 rows affected | unchanged | +| `DELETE FROM dbo.orders WHERE order_id = 3` | 0 rows affected | unchanged | + +Ground truth, read back with the tenant set to 2 as `dbo`: + +```text +order_id tenant_id amount +1 2 100.00 <- moved out of tenant 1 by tenant 1 +3 2 300.00 +4 2 400.00 +5 2 500.00 <- created by tenant 1 +``` + +A separate observation from a later run: a probe row inserted into another tenant during a failing +test could not be deleted afterwards by its author, because the filter predicate hides it from the +`DELETE` as well. It was recoverable only by disabling the policy. A cross-tenant write under a +filter-only policy is not just a leak, it is unrecoverable from the application side. + +Note also that the same filter applies to `dbo`. `SELECT COUNT(*) FROM dbo.orders` as `sa` with no +session context returned `0`. See [run 8](#run-8-who-bypasses-the-policy). + +## Run 2: the block predicate + +```sql +ALTER SECURITY POLICY sec.p_orders + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders AFTER INSERT, + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders AFTER UPDATE, + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders BEFORE UPDATE, + ADD BLOCK PREDICATE sec.fn_tenant(tenant_id) ON dbo.orders BEFORE DELETE; +``` + +The same two statements from run 1, as the same application user with tenant 1: + +| Statement | Result | +|---|---| +| `INSERT` a row stamped tenant 2 | `Msg 33504` | +| `UPDATE` my own row to tenant 2 | `Msg 33504` | +| `INSERT` a row stamped tenant 1 | succeeded | + +Full text: + +```text +Msg 33504: The attempted operation failed because the target object 'appdb.dbo.orders' has a +block predicate that conflicts with this operation. If the operation is performed on a view, the +block predicate might be enforced on the underlying table. Modify the operation to target only +the rows that are allowed by the block predicate. +``` + +## Run 3: AFTER INSERT without AFTER UPDATE + +`BEFORE UPDATE`, `BEFORE DELETE` and `AFTER UPDATE` dropped, leaving the filter predicate and +`AFTER INSERT`. As the application user with tenant 1: + +```sql +UPDATE dbo.orders SET tenant_id = 2 WHERE order_id = 1; +``` + +```text +UPDATE SUCCEEDED, rows = 1 +``` + +The row left the tenant. This is the reason both block operations are named separately in the +skill body: an author who reasons "the block predicate stops cross-tenant writes" and adds one has +closed half the hole. + +## Run 4: the duplicate key probe + +Filter predicate only, application user with tenant 1. Row 3 belongs to tenant 2. + +| Statement | Result | +|---|---| +| `SELECT COUNT(*) FROM dbo.orders WHERE order_id = 3` | `0`, correctly hidden | +| `INSERT INTO dbo.orders VALUES (3, 1, N'acme', 1.00)` | `Msg 2627`, violation of PRIMARY KEY, duplicate key value is (3) | +| `INSERT INTO dbo.orders VALUES (99, 1, N'acme', 1.00)` | succeeded | + +The policy hides rows. Constraints are evaluated on the whole table, so the difference between the +two attempts tells a caller whether an id exists in some other tenant. This is a property of unique +constraints rather than a defect in the policy, and the mitigation is an identifier with no +guessable ordering. + +## Run 5: session context under connection pooling + +Node client, one pool, `max` and `min` both set so the physical connections are stable. Each +"request" acquires a connection from the pool, runs one statement, and returns it. + +**Pool of 1, sequential.** The tenant id survived the request boundary: + +```text +request 1: sets tenant_id=1 spid=69 ctx=1 rows=2 ids=1,2 +request 2: sets NOTHING spid=69 ctx=1 rows=2 ids=1,2 +request 3: sets tenant_id=2 spid=69 ctx=2 rows=2 ids=3,4 +request 4: sets NOTHING spid=69 ctx=2 rows=2 ids=3,4 +``` + +Requests 2 and 4 read a tenant they never claimed to be, with no error. The client's pool did not +reset the session between acquisitions. `sp_reset_connection` cannot be called from T-SQL to check +this by hand: it is a protocol level reset and the attempt returned `Msg 208, Invalid object name`. +Whether a given driver issues it on reuse is a driver property, so the safe assumption is that the +context survives. + +**Pool of 5, 200 concurrent requests alternating tenant 1 and tenant 2.** Two shapes, same engine, +same policy, same second: + +| Shape | Cross-tenant reads | Empty results | +|---|---|---| +| `EXEC sp_set_session_context ...` as one pool request, `SELECT` as another | **103 of 200**, then **91 of 200** on a repeat | 0 | +| both in one batch on one acquired connection | **0 of 200**, twice | 0 | + +Nothing raised an error in either shape. The first shape is what an application produces when the +context is set by middleware holding a reference to the pool rather than to a connection. + +## Run 6: read only session context + +```sql +EXEC sp_set_session_context @key = N'tenant_id', @value = 1, @read_only = 1; +EXEC sp_set_session_context @key = N'tenant_id', @value = 2; +``` + +```text +Msg 15664: Cannot set key 'tenant_id' in the session context. The key has been set as read_only +for this session. +``` + +Replayed through the pool of 1: + +```text +req1 tenant 1 sets ctx read_only ctx=1 ids=1,2 +req2 tenant 2 sets ctx read_only (correctly) ERROR 15664 +req3 tenant 2 forgets ctx=1 ids=1,2 +``` + +`@read_only = 1` refused the correct caller and did not stop the incorrect one. It is safe only +where a connection is not reused across tenants. + +## Run 7: a database user per tenant + +```sql +CREATE TABLE sec.tenant_map (db_user sysname PRIMARY KEY, tenant_id int NOT NULL); +INSERT INTO sec.tenant_map VALUES (N'tenant_acme', 1), (N'tenant_globex', 2); +GO +CREATE FUNCTION sec.fn_tenant_by_user(@tenant_id int) +RETURNS TABLE +WITH SCHEMABINDING +AS +RETURN SELECT 1 AS ok FROM sec.tenant_map m + WHERE m.db_user = USER_NAME() AND m.tenant_id = @tenant_id; +``` + +| Check | Result | +|---|---| +| `tenant_acme` selects | rows 1 and 2 only | +| `tenant_globex` selects | rows 3 and 4 only | +| `tenant_acme` inserts a row stamped tenant 2 | `Msg 33504` | +| `tenant_acme` runs `sp_set_session_context @value = 2` then selects | still rows 1 and 2 | +| `tenant_acme` runs `EXECUTE AS USER = 'tenant_globex'` | `Msg 15517`, cannot execute as the database principal | +| 200 concurrent requests, one pool per tenant login | 0 cross-tenant reads | + +For contrast, under the session context predicate the same application user reset its own tenant id +and read the other tenant's rows, holding only `CONNECT` and the four table permissions. Nothing +had to be granted to call `sp_set_session_context`. + +## Run 8: who bypasses the policy + +All reads below are with no tenant set in session context. + +| Principal | `SELECT COUNT(*) FROM dbo.orders` | `ALTER SECURITY POLICY ... WITH (STATE = OFF)` | +|---|---|---| +| application user, `CONNECT` and four table permissions | `0` | `Msg 33268` | +| `sa`, which is `dbo` and a `db_owner` member | `0` | succeeded, then `4` | +| a user added to `db_owner` | `0` | succeeded, then `4` | +| a user with `ALTER ANY SECURITY POLICY` and `ALTER ON SCHEMA::sec` | `0` | succeeded, then `4` | +| the same user with `ALTER ANY SECURITY POLICY` only | `0` | `Msg 33268` | +| the same user with `ALTER ON SCHEMA::sec` only | `0` | `Msg 33268` | + +```text +Msg 33268: Cannot find the object "sec.p_orders" because it does not exist or you do not have +permissions. +``` + +Two results worth keeping. Nobody read through the policy, ownership included. And the permission +pair matters: the two halves were granted and revoked independently and each half alone produced +`Msg 33268`, which reports as a missing object rather than as a permission failure. + +## Run 9: present but not enforcing + +**`STATE = OFF`.** `sys.security_policies` reported `is_enabled = 0`. The application user with +tenant 1 set selected all four rows, no error, no warning. + +**A predicate that fails open.** + +```sql +CREATE FUNCTION sec.fn_tenant_failopen(@tenant_id int) +RETURNS TABLE WITH SCHEMABINDING AS +RETURN SELECT 1 AS ok + WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int) + OR SESSION_CONTEXT(N'tenant_id') IS NULL; +``` + +Installed with `ALTER SECURITY POLICY ... ALTER FILTER PREDICATE`. `is_enabled` stayed `1`. The +application user with **no** session context selected all four rows. No catalog view distinguishes +this from a correct policy. + +**A table the policy never covered.** `dbo.invoices` created with a `tenant_id` column and never +added to any policy. The application user with tenant 1 set: + +```text +orders (covered) 1|1 2|1 +invoices (NOT covered) 1|1 2|2 +``` + +The sweep query in section 5 of the skill body returned exactly this, at a moment when `orders` +carried one filter predicate and no block predicates: + +```text +table_name tenant_column filter_predicates block_predicates policy_enabled +[dbo].[invoices] tenant_id 0 0 NULL +[dbo].[orders] tenant_id 1 0 1 +``` + +`predicate_type` values were read back from `sys.security_predicates` and are `0` for `FILTER` and +`1` for `BLOCK`. + +The sweep query exactly as published in the skill body was then run against the full fixture, with +`dbo.orders` correctly configured: + +```text +table_name filter_predicates block_predicates policy_enabled +[dbo].[chunks] 1 0 1 +[dbo].[invoices] 0 0 NULL +[sec].[tenant_map] 0 0 NULL +``` + +`dbo.orders` was absent, which is the intended result. `sec.tenant_map` is the false positive to +expect: a mapping table carries a `tenant_id` column and needs no predicate. Dropping the +`AFTER UPDATE` block predicate from `dbo.orders` then made it appear with +`block_predicates = 1`, and restoring it removed it again. + +## Run 10: the vector index conflict + +A table of 300 rows with a `vector(4)` column. Both `SET QUOTED_IDENTIFIER ON` and +`SET ANSI_NULLS ON` are required, or `CREATE VECTOR INDEX` fails with `Msg 1934` about SET options, +which is a different failure and not the one being tested. A first attempt on a two row table +returned `Msg 42266`, at least 100 rows with non-null vectors are required. + +With a genuine index built: + +| Order | Statement | Result on the container | +|---|---|---| +| A | `CREATE VECTOR INDEX vi_chunks ON dbo.chunks(emb) WITH (METRIC='cosine', TYPE='diskann')` | created | +| A | `CREATE SECURITY POLICY sec.p_chunks ADD FILTER PREDICATE ... ON dbo.chunks` | `Msg 37579` | +| B | index dropped, `CREATE SECURITY POLICY` | created, `is_enabled = 1` | +| B | `CREATE VECTOR INDEX` | `Msg 42244` | + +```text +Msg 37579: The security policy 'sec.p_chunks' cannot reference tables with vector indexes. +Table 'dbo.chunks' has a vector index. + +Msg 42244: A vector index cannot be created on tables with security policies. +Table 'dbo.chunks' has security policy 'p_chunks'. +``` + +This reproduces the parity result already published in `rag-local-with-container`, on a fresh +container and a fresh table. + +**What does work locally.** With the policy on `dbo.chunks` and no vector index, exact search +returned correctly isolated results: + +```text +tenant 1 top-k chunk_id 150, 152, 148 all tenant_id 1 150 rows visible in total +tenant 2 top-k chunk_id 151, 149, 147 all tenant_id 2 +``` + +Both cloud coexistence claims in the table above are carried from the parity measurement in +`rag-local-with-container` and were not re-run here. Nothing else on this page depends on them. + +## Run 11: the isolation test, and the four planted failures + +The test in section 7 of the skill body was run as the application user against a correct policy +and printed five passes and `ALL PASS`, exit code `0`. + +Each failure was then planted one at a time, the test re-run, and the policy restored: + +| Planted | Output | Exit | +|---|---|---| +| `DROP BLOCK PREDICATE ON dbo.orders AFTER INSERT` | `FAIL 3: cross-tenant INSERT accepted` | 1 | +| `DROP BLOCK PREDICATE ON dbo.orders AFTER UPDATE` | `FAIL 4: tenant-move UPDATE accepted` | 1 | +| `WITH (STATE = OFF)` | `FAIL 1`, `FAIL 2`, `FAIL 4`, `FAIL 5` | 1 | +| `ALTER FILTER PREDICATE sec.fn_tenant_failopen(...)` | `FAIL 5: the predicate fails OPEN when no tenant is set` | 1 | + +The first version of the test used fixed probe ids and produced a false `FAIL 3: unexpected error +2627` on the run after any failing run, because a probe row written into the other tenant during +the failing run could no longer be seen or deleted by the test. That is the run 1 result showing up +inside the test itself. The fix is the randomised `@probe` in the published version, and the lesson +is that a test for a filter predicate cannot clean up rows the filter predicate hides from it. + +## Run 12: the published snippets, run verbatim + +The two SQL blocks in the skill body were extracted from the published file and run as written, +rather than retyped, because a snippet that was edited after it was measured is a snippet nobody +has run. + +| Snippet | Correct policy | With `AFTER UPDATE` dropped | +|---|---|---| +| the sweep query | `dbo.orders` absent, three candidates listed | `dbo.orders` listed, `block_predicates = 1` | +| the isolation test | `tenant isolation: ALL PASS`, exit `0` | `FAIL 4`, `Msg 50001`, exit `1` | + +## What is documented rather than measured + +- That row level security behaves identically in Azure SQL Database. Every run on this page is + against the local container. The predicate, block and permission mechanics are engine features + rather than container features, and nothing observed here contradicts the published behaviour, + but the cloud side was not re-run. +- The published guidance to avoid predicate functions that join extra tables, and to index the + columns a predicate filters on. Not measured here; this page makes no performance claim. +- That a policy applied to a view is enforced on the underlying table. This is stated inside the + `Msg 33504` text quoted above and was not separately exercised. + +## How to reproduce + +1. Start the Azure SQL Database container and create an application database. The engine does not + create one for you. +2. Create the login on `master` and the user in the application database. A contained user is + refused. +3. Apply the fixture above, then work down runs 1 to 4 with two client sessions. +4. For run 5, use any client that exposes an explicit pool size. Set the minimum and maximum to + the same number so the physical connections are stable, then issue the two shapes described. +5. For run 10, seed at least 100 rows with non-null vectors and set `QUOTED_IDENTIFIER` and + `ANSI_NULLS` on before creating the index. +6. Remove the container when finished. diff --git a/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/skill.spec.jsonc b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/skill.spec.jsonc new file mode 100644 index 0000000..13be7bc --- /dev/null +++ b/plugins/microsoft-sql-ssms/skills/rls-multi-tenant/skill.spec.jsonc @@ -0,0 +1,186 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "rls-multi-tenant", + "domain": "security-and-safe-sql", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked for tenant isolation on Azure SQL Database, an agent writes a security policy carrying a filter predicate and presents it as finished, because every SELECT then returns only the current tenant's rows and the demonstration looks correct. Measured on 2026-08-28 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure': with a filter predicate alone, a session holding tenant 1 inserted a row stamped tenant 2 and the statement succeeded, then updated one of its own rows into tenant 2 and that succeeded too, and both rows immediately vanished from the writer's view, so the application that created them could neither read them back nor delete them; recovery required disabling the policy. Adding the block predicate turned both into Msg 33504, and AFTER INSERT and AFTER UPDATE are separately required, because with only AFTER INSERT in place the tenant-move update still succeeded with no error. The second wrong belief is that SESSION_CONTEXT is a tenant boundary. It is per-connection state any caller on that connection can rewrite with no permission at all: a user holding only CONNECT plus SELECT, INSERT, UPDATE and DELETE on one table reset its own tenant id and read the other tenant's rows. It also outlives the request. With the context set in one pooled call and the query issued in another, 200 concurrent requests over a five connection pool returned the wrong tenant's rows 103 times, and 91 times on a repeat, with no error raised to any caller, while setting the context and querying on one acquired connection returned 0 wrong answers in 200 and a database user per tenant with one pool each also returned 0 in 200; sp_set_session_context with @read_only = 1 is not the fix, since on a reused connection it refused the next tenant's correct attempt with Msg 15664 while the forgetful path still leaked. Bypass is narrower than usually recalled and was confirmed rather than assumed: neither the table owner nor a db_owner member reads through the policy, both returning 0 rows with no tenant set, and what they hold is the ability to disable it, which requires ALTER ANY SECURITY POLICY together with ALTER on the schema holding the policy, since either permission alone failed with Msg 33268. A policy can also be present and not enforcing in three ways that all look healthy from the application, namely STATE = OFF, a predicate that returns a row when the tenant is unset, and a second tenant-scoped table nobody added to the policy. Being wrong costs a cross-tenant write no read can find and no application statement can undo, a tenant reading another tenant's rows about half the time under ordinary production concurrency with nothing raised and nothing logged, and an isolation control that ships never having been exercised beside its vector index, because the local container refuses to hold a security policy and a vector index on the same table in either order, Msg 37579 and Msg 42244.", + "value_probe": { + "gate": "The answer stops a wrong-tenant write at the database instead of only hiding rows from reads, and it ties the tenant value to the same connection that runs the query so a pooled connection cannot serve one tenant's value to another tenant's request. An answer that only adds application checks, or that only explains the read filtering, has not reached it.", + "task": "We run one Azure SQL Database for all our customers and every table carries a tenant_id column. Our Node API uses a connection pool:\n\n```js\nawait pool.request().query(`EXEC sp_set_session_context @key = N'tenant_id', @value = ${tenantId}`);\nconst rows = await pool.request().query('SELECT * FROM dbo.orders');\n```\n\nand in the database:\n\n```sql\nCREATE FUNCTION dbo.fn_tenant(@tenant_id int) RETURNS TABLE WITH SCHEMABINDING\nAS RETURN SELECT 1 AS ok WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int);\nGO\nCREATE SECURITY POLICY dbo.tenant_policy\nADD FILTER PREDICATE dbo.fn_tenant(tenant_id) ON dbo.orders\nWITH (STATE = ON);\n```\n\nIt demonstrates perfectly and it passed review. But one customer reported seeing another customer's orders once and we have never reproduced it, and our support team keeps finding orders sitting in the table that the owning tenant's application can neither see nor delete. Every tenant connects with the same database user.\n\nTell us what is wrong and exactly what to change.", + "states": [ + "The read filter alone does not stop a session writing or updating a row that carries another tenant's identifier, which is how the unreachable rows were created.", + "Refusing a wrong-tenant write means covering the insert case and the update case separately, because covering only one leaves the other silently allowed.", + "A row written with the wrong tenant identifier disappears from the writer's own view immediately, so the application cannot read it back or delete it afterwards.", + "Any caller on the connection can overwrite the session value with no special permission, so it is not a boundary against a caller who wants to cross it.", + "The tenant value has to be established and the query issued within one acquisition of the same pooled connection, or the value can be a previous request's." + ], + "refutes": [ + "A filter predicate prevents a session from inserting or updating a row stamped with another tenant's identifier, so no further predicate is needed.", + "The session value is per request state that the engine clears when the request or the batch ends, so a pooled connection cannot carry one tenant's value into another tenant's query.", + "Marking the session value read only makes it safe to reuse a pooled connection across tenants.", + "The table owner, or a member of db_owner, automatically reads through the policy and sees every tenant's rows.", + "The intermittent cross-tenant read has to be an application bug in how the tenant identifier is derived from the request, because the database side of this is already correct." + ] + }, + "posture": ["read", "write", "admin"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "entra-id-auth", + "design-azure-sql-schema", + "t-sql-correctness", + "rag-on-azure-sql", + "vector-search-azure-sql", + "rag-local-with-container", + "least-privilege-database-roles", + "permissions-and-privilege-review", + "multi-tenant-saas-patterns", + "secure-by-default-checklist" + ], + "validation": { + "target": "both", + "setup": [ + "a table carrying a non-nullable tenant column and rows belonging to at least two tenants", + "a login and a matching database user granted only CONNECT and the table permissions the application needs, never db_owner", + "a second tenant-scoped table deliberately left out of the policy, to exercise the sweep", + "a client that exposes an explicit connection pool size, so the pooling assertion can run concurrently", + "on the container, a chunk table with a vector column and no vector index" + ], + "assert": [ + "every tenant-scoped table carries a filter predicate and block predicates for both AFTER INSERT and AFTER UPDATE, and a generated policy with a filter predicate alone is flagged as unfinished", + "the predicate function is declared WITH SCHEMABINDING and returns no row when the tenant is unset, with no OR clause admitting an unset or administrative session", + "a cross-tenant insert and a tenant-move update are both refused with 33504, verified by running them rather than asserted in a comment", + "the statement setting the tenant and the statement reading run on one acquired connection, and the answer does not set the tenant through one pool call and query through another", + "the tenant is cleared or the connection is not reused before it returns to the pool, and no answer offers @read_only = 1 as the remedy for pooled reuse", + "no answer claims the table owner or db_owner reads through the policy, and any bypass discussion names the pairing of ALTER ANY SECURITY POLICY with ALTER on the policy's schema", + "a sweep over sys.security_predicates and sys.security_policies runs after any migration adding a table, and returns no tenant-scoped table lacking predicates or carrying a disabled policy", + "the isolation test runs as the application's own database user and has been observed failing with a predicate deliberately removed", + "no answer proposes creating a vector index on a table carrying a security policy on the container, and any local retrieval rehearsal uses exact distance search with no vector index" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_rls_filter_orders (order_id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL, amount decimal(10,2) NOT NULL);\nINSERT INTO dbo.probe_rls_filter_orders VALUES (1, 1, 100.00), (2, 1, 200.00), (3, 2, 300.00);\nEXEC sys.sp_executesql N'CREATE FUNCTION dbo.probe_rls_filter_fn(@tenant_id int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @tenant_id = CAST(SESSION_CONTEXT(N''tenant_id'') AS int);';\nGO\nGO\nCREATE SECURITY POLICY dbo.probe_rls_filter_pol\n ADD FILTER PREDICATE dbo.probe_rls_filter_fn(tenant_id) ON dbo.probe_rls_filter_orders\n WITH (STATE = ON);\nEXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1;\nINSERT INTO dbo.probe_rls_filter_orders VALUES (9, 2, 900.00);\nSELECT COUNT(*) AS visible_to_its_own_writer FROM dbo.probe_rls_filter_orders WHERE order_id = 9;", + "expect": { + "rows": [ + [ + 0 + ] + ] + }, + "why": "The measurement the whole skill turns on, made executable: with a filter predicate alone, a session holding tenant 1 writes a row stamped tenant 2, the statement succeeds with no error, and the row is then invisible to the application that created it. Measured 2026-08-28. Zero here is the leak, not the absence of one." + }, + { + "sql": "CREATE TABLE dbo.probe_rls_block_orders (order_id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL, amount decimal(10,2) NOT NULL);\nINSERT INTO dbo.probe_rls_block_orders VALUES (1, 1, 100.00), (3, 2, 300.00);\nEXEC sys.sp_executesql N'CREATE FUNCTION dbo.probe_rls_block_fn(@tenant_id int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @tenant_id = CAST(SESSION_CONTEXT(N''tenant_id'') AS int);';\nGO\nGO\nCREATE SECURITY POLICY dbo.probe_rls_block_pol\n ADD FILTER PREDICATE dbo.probe_rls_block_fn(tenant_id) ON dbo.probe_rls_block_orders,\n ADD BLOCK PREDICATE dbo.probe_rls_block_fn(tenant_id) ON dbo.probe_rls_block_orders AFTER INSERT,\n ADD BLOCK PREDICATE dbo.probe_rls_block_fn(tenant_id) ON dbo.probe_rls_block_orders AFTER UPDATE\n WITH (STATE = ON);\nEXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1;\nINSERT INTO dbo.probe_rls_block_orders VALUES (9, 2, 900.00);", + "expect": { + "error": 33504 + }, + "why": "The same cross-tenant insert, refused once the block predicate is in place. Measured 2026-08-28: Msg 33504, the target object has a block predicate that conflicts with this operation. Microsoft Learn states that security policies apply to all users including dbo, so the administrative connection a probe runs on is subject to it too." + }, + { + "sql": "CREATE TABLE dbo.probe_rls_halfblock_orders (order_id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL, amount decimal(10,2) NOT NULL);\nINSERT INTO dbo.probe_rls_halfblock_orders VALUES (1, 1, 100.00), (3, 2, 300.00);\nEXEC sys.sp_executesql N'CREATE FUNCTION dbo.probe_rls_halfblock_fn(@tenant_id int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @tenant_id = CAST(SESSION_CONTEXT(N''tenant_id'') AS int);';\nGO\nGO\nCREATE SECURITY POLICY dbo.probe_rls_halfblock_pol\n ADD FILTER PREDICATE dbo.probe_rls_halfblock_fn(tenant_id) ON dbo.probe_rls_halfblock_orders,\n ADD BLOCK PREDICATE dbo.probe_rls_halfblock_fn(tenant_id) ON dbo.probe_rls_halfblock_orders AFTER INSERT\n WITH (STATE = ON);\nEXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1;\nUPDATE dbo.probe_rls_halfblock_orders SET tenant_id = 2 WHERE order_id = 1;\nSELECT COUNT(*) AS still_visible_to_tenant_1 FROM dbo.probe_rls_halfblock_orders WHERE order_id = 1;", + "expect": { + "rows": [ + [ + 0 + ] + ] + }, + "why": "AFTER INSERT alone closes half the hole. Measured 2026-08-28: with AFTER UPDATE dropped, the tenant-move update succeeded with no error and the row left the tenant, which this probe reads back as the row no longer being visible to the session that owned it. The skill names both operations separately because of exactly this run." + }, + { + "sql": "EXEC sys.sp_set_session_context @key = N'probe_readonly_key', @value = 1, @read_only = 1;\nEXEC sys.sp_set_session_context @key = N'probe_readonly_key', @value = 2;", + "expect": { + "error": 15664 + }, + "why": "The remedy an agent offers for pooled reuse, and the measured reason it is not one: Msg 15664 refuses the next caller's CORRECT attempt to claim its own tenant, while the forgetful path that never sets the context at all sails past unaffected. Measured 2026-08-28. A key name of its own is used so the probe cannot poison another probe's session." + }, + { + "sql": "CREATE TABLE dbo.probe_rls_failopen_orders (order_id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL);\nINSERT INTO dbo.probe_rls_failopen_orders VALUES (1, 1), (2, 1), (3, 2), (4, 2);\nEXEC sys.sp_executesql N'CREATE FUNCTION dbo.probe_rls_failopen_fn(@tenant_id int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @tenant_id = CAST(SESSION_CONTEXT(N''tenant_id'') AS int) OR SESSION_CONTEXT(N''tenant_id'') IS NULL;';\nGO\nGO\nCREATE SECURITY POLICY dbo.probe_rls_failopen_pol\n ADD FILTER PREDICATE dbo.probe_rls_failopen_fn(tenant_id) ON dbo.probe_rls_failopen_orders\n WITH (STATE = ON);\nEXEC sys.sp_set_session_context @key = N'tenant_id', @value = NULL;\nSELECT\n (SELECT COUNT(*) FROM dbo.probe_rls_failopen_orders) AS rows_visible_with_no_tenant,\n (SELECT CAST(is_enabled AS int) FROM sys.security_policies WHERE name = 'probe_rls_failopen_pol') AS policy_enabled;", + "expect": { + "rows": [ + [ + 4, + 1 + ] + ] + }, + "why": "A policy that is present, enabled, schema bound, and enforcing nothing, because one OR clause admits an unset session. Measured 2026-08-28: all four rows returned with no tenant set while is_enabled stayed 1. No catalog view tells this apart from a correct policy, which is why the skill demands the predicate be read rather than the policy be listed." + } + , + { + "sql": "CREATE USER probe_r_a WITHOUT LOGIN;\nGO\nCREATE USER probe_r_b WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_r_a';\nEXECUTE AS USER = 'probe_r_b';\nREVERT;\nREVERT;", + "expect": { + "error": 15517 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Borrowing another tenant's identity is refused because impersonation is a permission the tenant does not hold. The nesting matters: run as sa the same statement succeeds, so the probe establishes the low privilege context first and the claim is about the tenant and not about the statement. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE SCHEMA probe_r_sec;\nGO\nCREATE TABLE dbo.probe_r_vec (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, tenant int NOT NULL, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_r_vec (tenant, v) SELECT 1, CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_r_vec_ix ON dbo.probe_r_vec (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nCREATE FUNCTION probe_r_sec.fn_probe_r (@t int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @t = 1;\nGO\nCREATE SECURITY POLICY probe_r_sec.p_probe_r ADD FILTER PREDICATE probe_r_sec.fn_probe_r(tenant) ON dbo.probe_r_vec WITH (STATE = ON);", + "expect": { + "error": 37579 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Row level security and a vector index do not coexist on the container, in this order. The skill's claim is that the isolation control and the retrieval index collide, so this asserts the collision rather than describing it. It goes red if the statement succeeds, or fails with any other number." + } + , + { + "sql": "CREATE SCHEMA probe_r_s2;\nGO\nCREATE TABLE dbo.probe_r_vi (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, tenant int NOT NULL, v vector(3) NOT NULL);\nGO\nINSERT INTO dbo.probe_r_vi (tenant, v) SELECT 1, CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nCREATE FUNCTION probe_r_s2.fn_vi (@t int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @t = 1;\nGO\nCREATE SECURITY POLICY probe_r_s2.p_vi ADD FILTER PREDICATE probe_r_s2.fn_vi(tenant) ON dbo.probe_r_vi WITH (STATE = ON);\nGO\nCREATE VECTOR INDEX probe_r_vi_ix ON dbo.probe_r_vi (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42244 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Policy first, then the index: the other order of the collision this skill records, and a different number from the index-first case. Both orders are refused, which is what makes the isolation control and the retrieval index a real choice rather than a sequencing problem. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_r_key (tenant int NOT NULL, k nvarchar(20) NOT NULL, CONSTRAINT pk_probe_r_key PRIMARY KEY (tenant, k));\nGO\nINSERT INTO dbo.probe_r_key VALUES (2, N'dup');\nGO\nINSERT INTO dbo.probe_r_key VALUES (2, N'dup');", + "expect": { + "error": 2627 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A filter predicate hides rows and not their existence. This is the number a tenant sees when it inserts a key another tenant already holds, and it is why the key space of another tenant is probeable through a filter policy alone. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE SCHEMA probe_r_s3;\nGO\nCREATE TABLE dbo.probe_r_pol (id int, tenant int);\nGO\nCREATE FUNCTION probe_r_s3.fn_pol (@t int) RETURNS TABLE WITH SCHEMABINDING AS RETURN SELECT 1 AS ok WHERE @t = 1;\nGO\nCREATE SECURITY POLICY probe_r_s3.p_pol ADD FILTER PREDICATE probe_r_s3.fn_pol(tenant) ON dbo.probe_r_pol WITH (STATE = ON);\nGO\nCREATE USER probe_r_holder WITHOUT LOGIN;\nGO\nGRANT ALTER ANY SECURITY POLICY TO probe_r_holder;\nGO\nEXECUTE AS USER = 'probe_r_holder';\nALTER SECURITY POLICY probe_r_s3.p_pol WITH (STATE = OFF);\nREVERT;", + "expect": { + "error": 33268 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Disabling a policy needs BOTH the database wide permission and ALTER on the schema holding it. This user has the first alone, and the refusal reads as a missing object rather than as a missing permission, which is the wording that sends someone looking for a typo. Measured alongside the schema-alone and the plain-user cases, which produce the same number. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Add row level security to this table so each tenant only sees its own rows", + "Write a security policy and predicate function for multi-tenant isolation on Azure SQL Database", + "Should I use SESSION_CONTEXT or a database user per tenant for tenant isolation", + "Write a test that proves one tenant cannot read another tenant's data", + "Do I need a block predicate as well as a filter predicate", + "Review this multi-tenant isolation design before it goes to production", + "Who can bypass a row level security policy in my database" + ], + "implicit": [ + "a customer reported seeing another customer's orders and we cannot reproduce it", + "our tenant filter works in testing and returns the wrong rows under load", + "the security policy is enabled but every tenant can still see everything", + "rows are being created that nobody in the application can find afterwards", + "the attempted operation failed because the target object has a block predicate", + "we added a new table last sprint and it is not being filtered by tenant", + "cannot set key in the session context, the key has been set as read_only for this session", + "how do I stop a tenant writing a row into someone else's tenant id", + "my retrieval returned a chunk that belongs to a different customer" + ], + "routing": [ + "make this shared-database SaaS schema safe for multiple customers", + "prove that our tenant separation actually works" + ], + "negative": [ + "How do I set up Microsoft Entra authentication for my Azure SQL database", + "Login failed for user when my app connects to Azure SQL", + "Encrypt this column so the database never sees the plaintext", + "Mask the last four digits of this card number in query results", + "How do I audit who read this table", + "Write a paginated query for Azure SQL Database", + "Create a vector index for my embeddings table in Azure SQL Database", + "How wide can an index key be on Azure SQL Database", + "Give each of my tenants a separate database instead of a shared one", + "Grant my reporting user read access to these three schemas" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings deleted file mode 120000 index ef276ee..0000000 --- a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings +++ /dev/null @@ -1 +0,0 @@ -../../../skills/azure-functions-sql-bindings \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/SKILL.md b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/SKILL.md new file mode 100644 index 0000000..5be7aea --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/SKILL.md @@ -0,0 +1,273 @@ +--- +name: azure-functions-sql-bindings +description: >- + Wires Azure Functions to Azure SQL Database with the SQL input and output bindings and the SQL + trigger, including the change tracking the trigger cannot run without and the identity + permissions the trigger needs beyond the ones the bindings need. Use when a user asks for "a + serverless CRUD API over SQL", to "add a SQL input binding", "write to SQL from a function", + "react to inserts and updates", "SQL trigger function", "SqlTrigger", "SqlInput", "SqlOutput", + or says "my SQL trigger never fires and there is no error". Also use when an output binding + silently updated an existing row instead of inserting, which is the documented upsert behaviour. + This is the Azure SQL Database story; the same bindings against the local Azure SQL Database + container belong to azuresql-db-functions, and connection reuse across invocations to the + per-language connect skills. +--- + +# Azure Functions with the Azure SQL bindings + +Two bindings and a trigger. The bindings are straightforward. **The trigger has a prerequisite +that fails without failing the app**, which is what this skill exists for. + +Every command and C# block below was measured 2026-09-03 against Core Tools 4.12.0, .NET SDK +8.0.421 and `Microsoft.Azure.Functions.Worker.Extensions.Sql` 3.1.536. Error numbers are +from Microsoft Learn. + +## What this skill does not own + +| Question | Skill | +|---|---| +| These bindings against the local Azure SQL Database container | `azuresql-db-functions` | +| Reusing a connection across invocations, pool sizing, retry | `connect-from-dotnet`, `connect-from-python`, `connect-from-typescript-and-node` | +| Getting the identity to a working passwordless connection | `entra-id-auth` | +| Publishing the app and its infrastructure | `deploy-app-to-azure` | + +The bindings open their own connections, so that pooling advice is about code you write inside the +function. + +## Step 0: scaffold and add the extension + +```bash +func init todo-fn --worker-runtime dotnet-isolated +cd todo-fn +dotnet add package Microsoft.Azure.Functions.Worker.Extensions.Sql +``` + +That resolved 3.1.536, clearing the v3.1.284 floor Learn sets for Consumption plans. Two +things the tooling does not tell you: + +**Core Tools 4.12.0 scaffolds `net10.0`.** On .NET SDK 8.0.421 the next restore fails with +`error NETSDK1045: The current .NET SDK does not support targeting .NET 10.0`. Set +`` to a version your SDK has before you add the package. + +**`func new` cannot create the SQL functions, though `func templates list` advertises them.** The +template name loses its spaces and is then rejected: + +```bash +func templates list | grep 'SQL Trigger' +func new --template "SQL Trigger" --name ToDoTrigger +``` + +The first prints `SQL Trigger`. The second fails with +`Unknown template 'SQLTrigger' (Parameter 'templateName')`, as do `SqlTrigger`, `SQLTrigger` and +`SQL Input Binding`. `HTTP trigger` succeeds in the same command, so a working `func new` proves +nothing about the SQL templates. Write the attribute by hand. + +## The failure this skill exists to prevent + +The trigger is built on SQL change tracking. At startup the listener asks the database for +`CHANGE_TRACKING_MIN_VALID_VERSION()`, which Learn documents as returning null when +change tracking is off for the database, the object id is invalid, or permission to the table is +insufficient. The listener throws on that null, before it polls once. + +**What that looks like from outside is nothing.** The host catches it and restarts that one +listener with exponential backoff, which Learn documents as startup retries. The app starts, +reports healthy, keeps serving every other function. The trigger never fires, and the only evidence +is one error line naming the listener rather than the missing feature. So change tracking is step +one, not a tuning step. + +## Step 1: enable change tracking, in two places + +Two statements. Enabling it on the database does **not** enable it on any table, and the table +statement fails if the database is not enabled first. + +```sql +ALTER DATABASE CURRENT +SET CHANGE_TRACKING = ON +(CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); + +ALTER TABLE [dbo].[ToDo] +ENABLE CHANGE_TRACKING; +``` + +`CHANGE_RETENTION` defaults to 2 days and is how far back the trigger catches up; an app stopped +longer loses the older changes. `AUTO_CLEANUP = OFF` pauses that removal while diagnosing a stalled +trigger. These options cannot be combined with other `ALTER DATABASE` options in one statement, +which fails with Msg 22114. + +**The monitored table must have a primary key.** Without one the second statement fails with +`Msg 4997, Cannot enable change tracking on table ''. Change tracking requires a primary key +on the table.` This skill printed Msg 22119 for that refusal until 2026-09-06, when it was measured +on the Azure SQL Database container at 18.0.226_4_147 and came back 4997. A deployment log grepped +for 22119 finds nothing. The cloud number is not verified here, so match on the text if you are +reading a log from a real Azure SQL Database. + +## Step 2: write the trigger + +The trigger receives a batch of changes, each carrying the changed item and the operation +(`Insert`, `Update`, `Delete`). This compiles as written: + +```csharp +[Function("ToDoTrigger")] +public static void Run( + [SqlTrigger("[dbo].[ToDo]", "SqlConnectionString")] + IReadOnlyList> changes, + FunctionContext context) +{ + foreach (SqlChange change in changes) + { + context.GetLogger("ToDoTrigger").LogInformation( + "{Operation} on {Id}", change.Operation, change.Item.Id); + } +} +``` + +Open [references/bindings-by-language.md](references/bindings-by-language.md) before writing this +declaration in anything but C#: the attribute name, argument casing and package change per +language, and nothing else does. + +**Changes are batched per row, not per statement.** A row written three times between two polls +arrives as one entry, showing the difference between the last processed state and now. +Anything needing every intermediate value needs another mechanism. + +## Step 3: grant the trigger more than the bindings need + +`db_datareader` and `db_datawriter` are enough for the input and output bindings. Microsoft's +wording is that they **are not sufficient** for the trigger, which reads change tracking and keeps +its own state and leases tables in an `az_func` schema it creates if absent. + +```sql +GRANT CREATE TABLE TO []; +GRANT CREATE SCHEMA TO []; + +GRANT SELECT ON [dbo].[ToDo] TO []; +GRANT VIEW CHANGE TRACKING ON [dbo].[ToDo] TO []; + +CREATE SCHEMA az_func; +GO +GRANT ALTER ON SCHEMA::az_func TO []; +GRANT SELECT, INSERT, UPDATE, DELETE ON SCHEMA::az_func TO []; +``` + +Missing these fails the same way: a listener that will not start, an app that looks fine. + +## Step 4: input and output bindings + +The input binding runs a query or stored procedure and hands the rows to the function. +**Parameters are bound, not interpolated**, which keeps a route value out of the SQL text: + +```csharp +[SqlInput( + commandText: "SELECT [Id], [title] FROM dbo.ToDo WHERE Id = @Id", + commandType: System.Data.CommandType.Text, + parameters: "@Id={Query.id}", + connectionStringSetting: "SqlConnectionString")] +IEnumerable todo +``` + +`parameters` is one string of the form `@a=1,@b=2`. No name or value may contain a comma or an +equals sign, a real limit binding free text. + +The output binding takes an object or a collection and a table name: + +```csharp +[SqlOutput("dbo.ToDo", connectionStringSetting: "SqlConnectionString")] +public ToDoItem ToDoItem { get; set; } +``` + +**It upserts.** Learn states the output bindings use T-SQL `MERGE`, so a write whose key already +exists updates that row rather than failing, and an agent treating the binding as an insert will +report a create that was an overwrite. `MERGE` also needs `SELECT` on the target, not only +`INSERT`. + +Two limits. Columns typed `NTEXT`, `TEXT` or `IMAGE` are unsupported and the upsert fails, because +they are incompatible with the `OPENJSON` the binding builds its `MERGE` from. And an +exception in the binding stops the function, so an HTTP trigger returns 500 unless +`IAsyncCollector` is used and `FlushAsync` awaited inside a try block. + +## Step 5: point it at Azure SQL Database, passwordless + +`connectionStringSetting` names an **application setting**, not a connection string. Locally it +lives in `local.settings.json`, which never goes into source control. In Azure it is an app setting +whose value carries an identity: + +```text +Server=.database.windows.net; Authentication=Active Directory Default; Database=; User Id= +``` + +Learn's managed identity tutorial states `Active Directory Default` resolves to developer +credentials locally and the managed identity in Azure, so one string serves both, and that +`User Id` is omitted for a system-assigned identity. The bindings overview page shows +`Active Directory Managed Identity` instead; both are valid, only the first also works locally. +The identity needs a database user: + +```sql +CREATE USER [] FROM EXTERNAL PROVIDER; +ALTER ROLE db_datareader ADD MEMBER []; +ALTER ROLE db_datawriter ADD MEMBER []; +GO +``` + +`` is the identity's own name for a user-assigned identity, the function app's name +for a system-assigned one. If a trigger is in play, add the grants from step 3 on top. + +## Step 6: tune the loop and let it scale + +`MaxBatchSize` defaults to 100, `PollingIntervalMs` to 1000 and `MaxChangesPerWorker` to 1000, all +under `extensions.Sql` in `host.json`. On a Premium plan, scaling out on pending change count needs +runtime scale monitoring: + +```bash +az resource update -g -n /config/web \ + --set properties.functionsRuntimeScaleMonitoringEnabled=1 \ + --resource-type Microsoft.Web/sites +``` + +A batch that throws is retried after 60 seconds; a row failing five times in a row is ignored for +all future changes. A poison row is skipped, not queued behind, so log the key on failure or the +loss is invisible. + +## Check it worked + +Change tracking is provable from SQL alone, before the app runs. Two rows back, one per query, or +the trigger does not start: + +```bash +sqlcmd -S .database.windows.net -d -G -Q \ + "SELECT DB_NAME() AS db, is_auto_cleanup_on, retention_period, retention_period_units_desc + FROM sys.change_tracking_databases WHERE database_id = DB_ID(); + SELECT OBJECT_NAME(object_id) AS tracked_table FROM sys.change_tracking_tables;" +``` + +An empty second result set is the common failure: the database statement looks done. Then confirm +what the listener reads, a number rather than `NULL` once both statements have run: + +```sql +SELECT CHANGE_TRACKING_MIN_VALID_VERSION(OBJECT_ID('dbo.ToDo')) AS min_valid_version; +``` + +Only then read the host log: `Started SQL trigger listener for table` is the success line, and +`Failed to start SQL trigger listener` on a healthy app is this skill's subject. Finally, +write the same primary key twice through the output binding. Two 201 responses and one row +confirms the `MERGE`. + +## Do not + +- Do not report a trigger as working because the function app started. It starts either way. +- Do not enable change tracking on the database and stop. The table statement is separate, and is + the one usually missed. +- Do not grant only `db_datareader` and `db_datawriter` to a trigger identity. +- Do not create the `az_func` tables by hand or rename them. The trigger owns them. +- Do not treat the output binding as an insert. It is a `MERGE` on the primary key. +- Do not reach for `func new` here, or read a name in `func templates list` as proof it scaffolds. +- Do not open your own connection per invocation. That belongs to the connect skills. + +## References + +- [references/bindings-by-language.md](references/bindings-by-language.md): read it when the + target language is not C#, or to look up the bundle and package a language needs. +- [Azure SQL bindings for Functions](https://learn.microsoft.com/azure/azure-functions/functions-bindings-azure-sql): + the authority. Fetch it rather than recalling it. +- [Azure SQL trigger for Functions](https://learn.microsoft.com/azure/azure-functions/functions-bindings-azure-sql-trigger): + fetch it for the host.json and local.settings.json tables and the retry rules. +- [About change tracking](https://learn.microsoft.com/sql/relational-databases/track-changes/about-change-tracking-sql-server): + retention, cleanup and what the change tables hold. diff --git a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/references/bindings-by-language.md b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/references/bindings-by-language.md new file mode 100644 index 0000000..688969e --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/references/bindings-by-language.md @@ -0,0 +1,183 @@ +# Binding declarations by language + +## Contents + +- [What you install](#what-you-install) +- [C#, isolated worker](#c-isolated-worker-model) +- [C#, in-process](#c-in-process-model) +- [Java](#java) +- [Python v2](#python-v2-programming-model) +- [JavaScript and TypeScript](#javascript-and-typescript) +- [PowerShell and function.json](#powershell-and-the-functionjson-form) +- [Property names in one table](#property-names-in-one-table) +- [Host settings](#host-settings) + +Verified against the Microsoft Learn binding reference on 2026-09-03, and the C# rows built on +Core Tools 4.12.0 with .NET SDK 8.0.421. The bindings require version 4.x or later of +the Functions runtime. Go support is not available for these bindings. + +## What you install + +| Language | What to add | +|---|---| +| C#, isolated worker | `dotnet add package Microsoft.Azure.Functions.Worker.Extensions.Sql` (3.1.536 on 2026-09-03) | +| C#, in-process | `dotnet add package Microsoft.Azure.WebJobs.Extensions.Sql` | +| Java | The extension bundle, plus `com.microsoft.azure.functions:azure-functions-java-library-sql` | +| Python, JavaScript, TypeScript, PowerShell | The extension bundle only | + +The extension bundle reference lives in `host.json`. This is what `func init --worker-runtime node` +wrote on Core Tools 4.12.0, measured 2026-09-03: + +```json +{ + "version": "2.0", + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} +``` + +Every function targeting the same database must use the same extension version. + +The in-process C# model reaches end of support on 10 November 2026, so new work belongs on the +isolated worker model. + +Core Tools 4.12.0 scaffolds none of these: `func new --template "SQL Trigger"` fails with +`Unknown template 'SQLTrigger' (Parameter 'templateName')` though `func templates list` prints the +name. Write the declaration below by hand. + +## C#, isolated worker model + +```csharp +// input +[SqlInput( + commandText: "SELECT [Id], [title] FROM dbo.ToDo WHERE Id = @Id", + commandType: System.Data.CommandType.Text, + parameters: "@Id={Query.id}", + connectionStringSetting: "SqlConnectionString")] +IEnumerable todo + +// output, on a property of a multiple-output return type +[SqlOutput("dbo.ToDo", connectionStringSetting: "SqlConnectionString")] +public ToDoItem ToDoItem { get; set; } + +// trigger +[SqlTrigger("[dbo].[ToDo]", "SqlConnectionString")] +IReadOnlyList> changes +``` + +## C#, in-process model + +Both the input and the output binding are the single `[Sql]` attribute; only the direction of the +parameter distinguishes them. The trigger keeps its own name. + +```csharp +[Sql(commandText: "dbo.ToDo", connectionStringSetting: "SqlConnectionString")] +IAsyncCollector toDoItems +``` + +## Java + +Annotations from `com.microsoft.azure.functions.sql.annotation`, all requiring `name`: + +```java +@SQLInput(name = "todo", commandText = "SELECT * FROM dbo.ToDo WHERE Id = @Id", + commandType = "Text", parameters = "@Id={Query.id}", + connectionStringSetting = "SqlConnectionString") + +@SQLOutput(name = "toDoItem", commandText = "dbo.ToDo", + connectionStringSetting = "SqlConnectionString") + +@SQLTrigger(name = "todoItems", tableName = "[dbo].[ToDo]", + connectionStringSetting = "SqlConnectionString") +``` + +Note the casing: `SQL` in Java, `Sql` in C#. + +## Python, v2 programming model + +Decorators on the function app object, with snake_case arguments: + +```python +@app.sql_input(arg_name="todo", + command_text="SELECT * FROM dbo.ToDo WHERE Id = @Id", + command_type="Text", + parameters="@Id={id}", + connection_string_setting="SqlConnectionString") + +@app.sql_output(arg_name="todo", + command_text="[dbo].[ToDo]", + connection_string_setting="SqlConnectionString") + +@app.sql_trigger(arg_name="todo", + table_name="ToDo", + connection_string_setting="SqlConnectionString") +``` + +Rows are `func.SqlRow` and `func.SqlRowList`; `func.SqlRow.from_dict` builds one from a parsed +body. + +## JavaScript and TypeScript + +In the v4 model the bindings are `input.sql()` and `output.sql()`, taking an options object with +`commandText`, `commandType`, `parameters` and `connectionStringSetting`. The v3 model uses the +`function.json` form below. + +## PowerShell and the function.json form + +```json +{ + "name": "todoItems", + "type": "sql", + "direction": "out", + "commandText": "dbo.ToDo", + "connectionStringSetting": "SqlConnectionString" +} +``` + +```json +{ + "name": "todoChanges", + "type": "sqlTrigger", + "direction": "in", + "tableName": "dbo.ToDo", + "connectionStringSetting": "SqlConnectionString" +} +``` + +The property table below carries the `type` and `direction` values. In PowerShell the value +reaches the binding through `Push-OutputBinding -Name `. + +## Property names, in one table + +| Concept | C# isolated | C# in-process | Java | Python v2 | function.json | +|---|---|---|---|---|---| +| Input | `SqlInput` | `Sql` | `SQLInput` | `sql_input` | `type: sql, direction: in` | +| Output | `SqlOutput` | `Sql` | `SQLOutput` | `sql_output` | `type: sql, direction: out` | +| Trigger | `SqlTrigger` | `SqlTrigger` | `SQLTrigger` | `sql_trigger` | `type: sqlTrigger` | +| Query or table | `commandText` | `commandText` | `commandText` | `command_text` | `commandText` | +| Text or procedure | `commandType` | `commandType` | `commandType` | `command_type` | `commandType` | +| Bound parameters | `parameters` | `parameters` | `parameters` | `parameters` | `parameters` | +| Monitored table | `TableName` | `TableName` | `tableName` | `table_name` | `tableName` | +| Setting name | `connectionStringSetting` | `connectionStringSetting` | `connectionStringSetting` | `connection_string_setting` | `connectionStringSetting` | + +`commandType` is `Text` for a query and `StoredProcedure` for a procedure. `parameters` is one +string in the form `@a=1,@b=2`, and neither a name nor a value may contain a comma or an equals +sign. The trigger also accepts an optional leases table name; leave it unset unless two functions +must watch the same table. + +## Host settings + +Under `extensions.Sql` in `host.json`: + +| Setting | Default | Effect | +|---|---|---| +| `MaxBatchSize` | 100 | Changes handed to the function per iteration | +| `PollingIntervalMs` | 1000 | Delay between batches | +| `MaxChangesPerWorker` | 1000 | Pending changes per worker before scaling out, with runtime scale monitoring on | + +The same three exist under `Sql_Trigger_` names in `local.settings.json` for local runs. +**Microsoft Learn contradicts itself on the first**: its settings table says +`Sql_Trigger_BatchSize`, its own example on the same page says `Sql_Trigger_MaxBatchSize`. Neither +is measured here, so set it and confirm the batch size changed rather than trusting a spelling. diff --git a/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/skill.spec.jsonc new file mode 100644 index 0000000..33577bc --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/azure-functions-sql-bindings/skill.spec.jsonc @@ -0,0 +1,201 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "azure-functions-sql-bindings", + "domain": "apis-and-application-patterns", + "value": ["not-in-training-data", "multi-step-with-silent-failures"], + "correction": "Agents treat change tracking as an optimisation the Azure SQL trigger can run without, when it is the mechanism the trigger is built on: the listener asks the database for the change tracking minimum valid version of the monitored table, gets a null back when tracking is off, and throws before it ever polls. The Functions host then retries that one listener in the background with exponential backoff, so the app starts, stays running, keeps serving every other function, and the trigger never fires while the only sign of the problem is a listener startup error buried in the host log. The identity permissions are wrong for the same reason: db_datareader and db_datawriter are enough for the input and output bindings and are explicitly not enough for the trigger, which also needs to read change tracking and to create its own state tables.", + "value_probe": { + "gate": "The answer identifies a database-side prerequisite the monitored table must carry before this can ever fire, and treats a healthy app whose other functions work as entirely consistent with this one never starting.", + "task": "I wrote a function that is meant to run whenever a row changes in one of our Azure SQL Database tables. The function app deploys, starts, and every other function in the same app works normally. This one has never run once, and I cannot find an error anywhere obvious in the logs. I have updated rows by hand to test it and nothing happens. The app connects with its managed identity, which we granted read and write on the database. What is wrong and how do I confirm it?", + // 2026-09-19: the first statement used to read "A tracking mechanism has to be enabled on the + // monitored table, because it is what the trigger reads to find changes." It measured the + // change tracking prerequisite, which the model already asserts 8 of 10 times with no skill + // loaded, so it had no room to show the skill adding anything and could only fall. The + // prerequisite itself is still enforced, by the gate and by three of the four refutes + // statements. The replacement names the mechanism behind it: no answer in the ten no-skill + // runs named the minimum valid version call, while all ten skill-loaded answers did. + "states": [ + "The listener's startup check asks the database for the monitored table's minimum valid change tracking version, and that call answers with nothing rather than an error when the prerequisite is missing, which is what the listener throws on.", + "The listener fails at startup and is retried in the background, so the app stays up and every other function keeps serving.", + "This trigger needs more than reading and writing data, including the ability to create and maintain its own state tables." + ], + "refutes": [ + "The trigger works without any change tracking enabled, so the fault must be the connection string or the binding configuration.", + "If the trigger's listener had failed, the function app would fail to start or would surface an obvious error.", + "Read and write permissions on the database are sufficient for this trigger to operate.", + "Enabling tracking is an optimisation that makes the trigger more efficient rather than a prerequisite." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db"], + "related": [ + "azuresql-db-functions", + "build-app-on-azure-sql", + "dab-rest-and-graphql", + "connect-from-dotnet", + "connect-from-python", + "connect-from-typescript-and-node", + "entra-id-auth", + "deploy-app-to-azure", + "connect-to-azure-sql" + ], + "validation": { + "target": "both", + "setup": [ + "an Azure SQL Database with a table that has a primary key", + "a function app on the isolated worker model with the Azure SQL extension referenced" + ], + "assert": [ + "change tracking is enabled at the database and again on the monitored table, as two separate statements", + "with change tracking off, the app starts and reports healthy while the trigger never fires, and the only evidence is a listener startup error in the host log", + "the trigger's identity holds more than db_datareader and db_datawriter: it can read change tracking on the table and create the state schema and tables", + "the output binding upserts on the primary key rather than inserting, so a repeated write updates the existing row instead of failing", + "the connection setting names an app setting, and the deployed value carries an identity rather than a password" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_ct_two_step (Id int NOT NULL PRIMARY KEY, title nvarchar(50) NOT NULL);\nIF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID())\n EXEC(N'ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);');\nGO\nSELECT (SELECT COUNT(*) FROM sys.change_tracking_databases WHERE database_id = DB_ID()) AS database_tracked,\n (SELECT COUNT(*) FROM sys.change_tracking_tables WHERE object_id = OBJECT_ID('dbo.probe_ct_two_step')) AS table_tracked;", + "expect": { + "rows": [ + [ + 1, + 0 + ] + ] + }, + "why": "The skill's first claim made executable: enabling change tracking at the database level does NOT enable it on any table. One row in sys.change_tracking_databases and zero in sys.change_tracking_tables is the exact state a developer is in when they run the ALTER DATABASE from Microsoft Learn, stop there, and then cannot work out why the SQL trigger listener never starts. Re-running the ALTER DATABASE is a no-op, so this probe does not care what order it runs in." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID())\n EXEC(N'ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);');\nGO\nCREATE TABLE dbo.probe_ct_enabled (Id int NOT NULL PRIMARY KEY, title nvarchar(50) NOT NULL);\nALTER TABLE dbo.probe_ct_enabled ENABLE CHANGE_TRACKING;\nGO\nSELECT (SELECT COUNT(*) FROM sys.change_tracking_tables WHERE object_id = OBJECT_ID('dbo.probe_ct_enabled')) AS table_tracked,\n CAST(CASE WHEN CHANGE_TRACKING_MIN_VALID_VERSION(OBJECT_ID('dbo.probe_ct_enabled')) IS NOT NULL THEN 1 ELSE 0 END AS int) AS listener_reads_a_version;", + "expect": { + "rows": [ + [ + 1, + 1 + ] + ] + }, + "why": "The other half of the same claim, and the one that proves the fix works: once BOTH statements have run, the table appears in sys.change_tracking_tables and CHANGE_TRACKING_MIN_VALID_VERSION returns a version rather than null. That function is what the SQL trigger listener calls at startup, and Microsoft Learn documents it as returning null when change tracking is off for the database, the object id is invalid, or permission to the table is insufficient. A non-null version here is the listener's own precondition, measured directly instead of inferred from a host log." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID())\n EXEC(N'ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);');\nGO\nCREATE TABLE dbo.probe_ct_no_pk (Id int NOT NULL, title nvarchar(50) NOT NULL);\nGO\nALTER TABLE dbo.probe_ct_no_pk ENABLE CHANGE_TRACKING;", + "expect": { + "error": 4997 + }, + "why": "The primary key prerequisite, which the skill states and which is otherwise only visible as a listener that quietly never starts. THE NUMBER HERE WAS WRONG UNTIL IT WAS MEASURED. It read Msg 22119, taken from Microsoft Learn's 22000 to 22999 error table rather than from a run, and the first live run of this probe returned Msg 4997 instead. Both messages exist on the engine and they are not the same claim: 4997 is 'Change tracking requires a primary key on the table. Create a primary key on the table before enabling change tracking' and is what a table with NO primary key gets, which is the case this probe builds; 22119 is 'requires a primary key constraint on the table to be enabled. Enable the primary key constraint' and is what a table whose primary key index has been disabled gets. Both measured 2026-09-05 on the container at build 18.0.226_4_147, the second by creating a nonclustered primary key and running ALTER INDEX ... DISABLE against it, and both read back from sys.messages. The next probe asserts the disabled-constraint half so the pair stays distinguishable." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID())\n EXEC(N'ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);');\nGO\nCREATE TABLE dbo.probe_ct_disabled_pk (Id int NOT NULL CONSTRAINT pk_probe_ct_disabled_pk PRIMARY KEY NONCLUSTERED, title nvarchar(50) NOT NULL);\nGO\nALTER INDEX pk_probe_ct_disabled_pk ON dbo.probe_ct_disabled_pk DISABLE;\nGO\nALTER TABLE dbo.probe_ct_disabled_pk ENABLE CHANGE_TRACKING;", + "expect": { + "error": 22119 + }, + "why": "The other half of the primary key prerequisite, and the reason the probe above had the wrong number for two weeks. A table that HAS a primary key whose index has been disabled is refused with Msg 22119, 'Change tracking requires a primary key constraint on the table to be enabled', which is a different sentence and a different repair from the Msg 4997 a table with no primary key gets. Measured 2026-09-05 on the container. It is worth its own round trip because the two are one letter apart in a reader's memory and the fix differs: create a key, versus rebuild an index that already exists. A disabled primary key is not a hypothetical here, it is what a large index rebuild leaves behind if it fails partway." + }, + { + "sql": "CREATE TABLE dbo.probe_out_upsert (Id int NOT NULL PRIMARY KEY, title nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_out_upsert (Id, title) VALUES (1, N'first');\nMERGE dbo.probe_out_upsert AS t\nUSING (SELECT * FROM OPENJSON(N'[{\"Id\":1,\"title\":\"second\"}]') WITH (Id int, title nvarchar(50))) AS s\nON t.Id = s.Id\nWHEN MATCHED THEN UPDATE SET t.title = s.title\nWHEN NOT MATCHED THEN INSERT (Id, title) VALUES (s.Id, s.title);\nSELECT COUNT(*) AS row_count, MAX(title) AS title FROM dbo.probe_out_upsert;", + "expect": { + "rows": [ + [ + 1, + "second" + ] + ] + }, + "why": "The output binding's upsert, written the way the binding writes it. Microsoft Learn states the output bindings use the T-SQL MERGE statement, and that the unsupported NTEXT, TEXT and IMAGE column types fail because they are incompatible with OPENJSON, which names both halves of the generated statement. Writing a row whose primary key already exists therefore updates it: one row back holding the second title, not a duplicate key error. An agent that reports this as a create has reported an overwrite." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID())\n EXEC(N'ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);');\nGO\nCREATE TABLE dbo.probe_perm_todo (Id int NOT NULL PRIMARY KEY, title nvarchar(50) NOT NULL);\nALTER TABLE dbo.probe_perm_todo ENABLE CHANGE_TRACKING;\nCREATE USER probe_binding_identity WITHOUT LOGIN;\nALTER ROLE db_datareader ADD MEMBER probe_binding_identity;\nALTER ROLE db_datawriter ADD MEMBER probe_binding_identity;\nGO\nEXECUTE AS USER = 'probe_binding_identity';\nGO\nSELECT HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'CREATE TABLE') AS can_create_table,\n HAS_PERMS_BY_NAME('dbo.probe_perm_todo', 'OBJECT', 'SELECT') AS can_select,\n HAS_PERMS_BY_NAME('dbo.probe_perm_todo', 'OBJECT', 'VIEW CHANGE TRACKING') AS can_view_change_tracking;\nGO\nREVERT;", + "expect": { + "rows": [ + [ + 0, + 1, + 0 + ] + ] + }, + "why": "Microsoft's claim that db_datareader and db_datawriter are enough for the input and output bindings and are NOT enough for the trigger, turned into a measurement instead of a quotation. The middle column is the control and must be 1, because db_datareader grants SELECT, which proves the impersonation took effect rather than the probe silently running as the administrator. The two zeroes either side are the trigger's missing prerequisites: CREATE TABLE for its az_func state tables, and VIEW CHANGE TRACKING on the monitored table. EXECUTE AS, SELECT, REVERT is the shape Learn's own HAS_PERMS_BY_NAME example uses, and REVERT returns no result set, so the SELECT is the last one." + }, + { + "where": "host", + "cmd": "func --version", + "expect": { + "succeeds": true + }, + "why": "Azure Functions Core Tools is the tool this skill's step 0 tells the reader to run, so a red here means the body opens with a command that is not there. Measured 2026-09-03 on the authoring host: exit 0, 4.12.0. Core Tools is a client tool a developer installs and is never shipped in the engine image, so this is a host probe." + }, + { + "where": "host", + "cmd": "func templates list | grep -q 'SQL Trigger'", + "expect": { + "succeeds": true + }, + "why": "Half of the tooling correction this skill carries: the SQL Trigger template IS advertised by Core Tools. Measured 2026-09-03, func templates list prints SQL Input Binding, SQL Output Binding and SQL Trigger across five worker languages. This probe and the next are only meaningful as a pair, because the finding is the gap between them." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && printf '{\"version\":\"2.0\"}' > \"$d/host.json\" && printf '{\"IsEncrypted\":false,\"Values\":{\"FUNCTIONS_WORKER_RUNTIME\":\"dotnet-isolated\"}}' > \"$d/local.settings.json\" && cd \"$d\" && func new --template \"SQL Trigger\" --name ProbeTrigger &1 | grep -q \"Unknown template 'SQLTrigger'\"", + "expect": { + "succeeds": true + }, + "why": "The other half, and the reason this skill tells the reader to write the attribute by hand. Measured 2026-09-03 on Core Tools 4.12.0: the template name loses its spaces and func new then rejects it with Unknown template 'SQLTrigger' (Parameter 'templateName'), while HTTP trigger succeeds in the same command. The probe builds its own minimal project because func new prompts for a worker runtime when local.settings.json does not declare one, and a prompting probe hangs a runner rather than failing it; the redirect from /dev/null is the second guard. A red here means Core Tools fixed it and step 0 should be rewritten." + }, + { + "where": "host", + "cmd": "dotnet --version", + "expect": { + "succeeds": true + }, + "why": "The .NET SDK is the second command in step 0, the one that adds Microsoft.Azure.Functions.Worker.Extensions.Sql. Measured 2026-09-03 on the authoring host: exit 0, 8.0.421. That SDK is also why the skill warns about NETSDK1045, because Core Tools 4.12.0 scaffolds net10.0 and this SDK cannot build it." + } + , + { + "sql": "ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS), AUTO_SHRINK OFF;", + "expect": { + "error": 22114 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Change tracking options cannot share an ALTER DATABASE statement with anything else, which is why the enable step is its own statement and not folded into a provisioning script. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_af_nopk (id int NOT NULL, v int);\nGO\nALTER TABLE dbo.probe_af_nopk ENABLE CHANGE_TRACKING;", + "expect": { + "error": 4997 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. THE CORRECTION. The monitored table must have a primary key, and this skill printed Msg 22119 for that refusal until 2026-09-06. The engine raises Msg 4997, with the text this skill quotes. A reader grepping a deployment log for 22119 finds nothing. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.change_tracking_databases WHERE database_id = DB_ID()) ALTER DATABASE CURRENT SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);\nGO\nCREATE TABLE dbo.probe_af_pk (id int NOT NULL PRIMARY KEY, v int);\nGO\nALTER TABLE dbo.probe_af_pk ENABLE CHANGE_TRACKING;", + "expect": { + "succeeds": true + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The control for the two refusals above: with a primary key and change tracking on at the database level, the enable succeeds. Without this the pair could both be passing for a reason that has nothing to do with the primary key. It goes red if the statement raises." + } + ] + }, + "triggering": { + "explicit": ["Use the azure-functions-sql-bindings skill"], + "implicit": [ + "add a SQL input binding to my Azure Functions app", + "write to Azure SQL from an Azure Function without writing ADO code", + "run a function when a row changes in my Azure SQL table", + "my SQL trigger function never fires and there is no error", + "how do I enable change tracking for the Azure Functions SQL trigger" + ], + "routing": [ + "build a serverless CRUD API over these tables", + "react to inserts and updates in the database", + "the function app is healthy but nothing is being processed", + "why did my second POST overwrite the first row instead of failing" + ], + "negative": [ + "use Azure Functions with the local Azure SQL Database container", + "set up the SQL trigger against my local container", + "expose these tables as REST and GraphQL with Data API builder", + "run Data API builder against my local Azure SQL Database container", + "reuse a connection pool across serverless invocations in Node.js", + "install the ODBC driver for Python in my function image", + "scaffold a new application on Azure SQL" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql deleted file mode 120000 index 788abb5..0000000 --- a/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/build-app-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/SKILL.md new file mode 100644 index 0000000..a1e584d --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/SKILL.md @@ -0,0 +1,335 @@ +--- +name: build-app-on-azure-sql +description: >- + Sequences standing up a new application on Azure SQL Database and routes each decision to the + skill that owns it. Use at the start of a project, when a user says "build an app on Azure + SQL", "add Azure SQL Database to my app", "which stack should I use with Azure SQL", "where do + I start", "generate an API over my database", or "get this into Azure without a password in the + repo". It establishes the order that works: server-side prerequisites before any code, an + identity instead of a password, schema through a migration not at startup, and a generated data + API before hand-written CRUD. It routes rather than repeats: provision-azure-sql-db creates the + database, connect-to-azure-sql drivers and retry, entra-id-auth the identity, + dab-rest-and-graphql and azure-functions-sql-bindings the API layer, deploy-app-to-azure + shipping, azuresql-db-scaffold the local container version. +--- + +# Build an application on Azure SQL Database + +**This is a router, not a tutorial.** It owns the order the work happens in and the four decisions +that get made wrong at the start. Every technique it names belongs to another skill, and the handoff +is the point. + +Verified 2026-09-03 against Azure CLI 2.90.0, go-sqlcmd 1.10.0, Data API builder 2.0.9 and .NET SDK +8.0.421, and against the four first-party Azure SQL Database quickstarts on Microsoft Learn. The two +sqlcmd builds were compared again on 2026-09-05, go-sqlcmd 1.10.0 against ODBC sqlcmd 18.6.0002.1. + +## The failure this exists to prevent + +Asked to build an application on Azure SQL Database, an agent starts inside the application: it +picks a stack, writes a data layer, and puts a user name and password in configuration. + +**Every first-party Azure SQL Database quickstart starts outside the application.** All four open +with the same section, before a line of code, and it configures the server rather than the project. +The three things that section does are the three things that fail silently: + +| Prerequisite | How it fails | +|---|---| +| A firewall rule covering the client address | Learn documents a latency of **up to** five minutes on security setting changes. **Measured 2026-09-05 it was immediate**, so do not plan around a five minute wait and do not read a refusal as a rule that has not landed yet: see [how long a rule really takes](#how-long-a-firewall-rule-really-takes). Network address translation also means the address the client connects from is often not the one in its own network configuration | +| A Microsoft Entra administrator on the logical server | Passwordless auth is simply off. The failure arrives at connect time as a login failure, which reads as a wrong credential | +| A database user for the deployed identity | Nothing at deploy time reports it missing. The application returns a server error, and its code is correct | + +None of these is a code bug, and all three present as one. + +## Step 0: prove the prerequisites, then write code + +Both commands are read-only. Creating the server is `provision-azure-sql-db` and configuring the +identity is `entra-id-auth`. + +```bash +az sql server firewall-rule list -g -s -o table +az sql server ad-admin list -g -s -o table +``` + +Then take one round trip to the database before an application exists, so that every later failure +is a failure of the code: + +```bash +az login +sqlcmd -S .database.windows.net,1433 -d -G -N mandatory -l 30 -m-1 \ + -Q "SELECT SUSER_NAME() AS connected_as, DB_NAME() AS db" +``` + +`-G` is Microsoft Entra authentication and takes no `-U`. Open `connect-to-azure-sql` when that +command fails, or before turning it into a connection string: it owns the switch-to-keyword +mapping, encryption, retry and pool sizing. + +### Which sqlcmd you have decides what you see + +Two different programs are called `sqlcmd`, and the message numbers in this skill and in every +skill it routes to come out of only one of them. Microsoft Learn calls them +[the two variants](https://learn.microsoft.com/sql/tools/sqlcmd/sqlcmd-utility#sqlcmd-variants): + +| | ODBC `sqlcmd` | Go `sqlcmd`, also written go-sqlcmd | +|---|---|---| +| Where it comes from | the Microsoft command line utilities, `mssql-tools18`, and the Azure SQL Database container image | a standalone download, `winget install sqlcmd`, `brew install sqlcmd` | +| Version it reports | 18.x | 1.x | +| How to tell | `sqlcmd -?` starts `Microsoft (R) SQL Server Command Line Tool` | `sqlcmd --version` prints a version alone | + +`-m-1` is an **ODBC `sqlcmd`** instruction, and it is the flag that makes a severity 10 message +print its `Msg` number, which that build otherwise does not do. Measured 2026-09-05 on 18.6.0002.1: +`RAISERROR('a severity ten note', 10, 1)` prints `Msg 50000, Level 0, State 1` at `-m-1` and the +bare text without it. + +**go-sqlcmd 1.10.0 does not do this.** Measured the same day against the same engine, it prints the +`Msg` header only at severity 11 and above, and a severity 10 message arrives as text with no number +at `-m-1`, `-m0` and `-m1` alike, on stdout and on stderr. Its `-m` still filters, so `-m17` drops a +severity 16 message entirely, but no value of it puts a number back on a warning. If the numbers a +skill tells you to look for never appear, check which build you are running before you conclude the +engine stayed quiet: query `sys.messages`, or run the statement through the ODBC build. + +Neither build turns a severity 10 message into a non-zero exit. `-b` returns 1 only above severity +10, so a warning never fails the command on either. + +Two facts decide what to do when the rule looks right and the connection still fails: + +- The five minute latency is a **cache**, and it is a ceiling rather than a wait to plan around. + `DBCC FLUSHAUTHCACHE` on the user database forces a refresh rather than waiting. It needs the + admin account or `KILL DATABASE CONNECTION`, and it needs a connection that already works, so it + cannot rescue the connection the firewall is refusing. +- A rule from `0.0.0.0` to `0.0.0.0` is the documented special case meaning Azure-internal traffic, + not a wildcard for the internet, and any virtual machine in Azure may then attempt to connect. + `deploy-app-to-azure` owns what the first-party templates put there. + +### How long a firewall rule really takes + +Learn says security setting changes take **up to** five minutes. Up to is the whole sentence, and +this skill used to repeat the ceiling as the expected case and tell readers the first run would +fail because of it. It does not. + +Measured 2026-09-05 against a Basic tier database on a logical server in `eastus2`, over three +trials in both directions. Each trial deleted the rule admitting the client and polled until the +connection was refused, then created it again and polled until the connection succeeded, with the +clock started the instant the `az` control-plane call returned and a poll every two seconds. + +| Trial | Rule removed, until refused | Rule created, until admitted | +|---|---|---| +| 1 | 1.0 s | 0.8 s | +| 2 | 0.8 s | 0.9 s | +| 3 | 0.8 s | 0.9 s | + +Every poll in every trial found the change already in effect on the **first** attempt, so the real +figure is below the measurement floor: one Entra `sqlcmd` login, about 0.8 s. Read the table as +"not more than a second", not as "about a second". + +What this changes in practice. **A connection refused straight after `az sql server firewall-rule +create` returned is very unlikely to be a rule that has not propagated yet, so do not wait five +minutes and try again.** Check the address first: network address translation means the address the +server sees is often not the one the client believes it has, and the refusal message names the +address the server saw. + +```output +Cannot open server 'example' requested by the login. Client with IP address '203.0.113.7' is not +allowed to access the server. ... It may take up to five minutes for this change to take effect. +``` + +That trailing sentence is part of the server's own message and is where the expectation comes from. +It is a ceiling in the message too. + +**One trap worth knowing before you diagnose a firewall at all.** On a server with Entra-only +authentication, a SQL authentication attempt is refused for being SQL authentication, before the +firewall is consulted: + +```output +Login failed for user 'someone'. Reason: Azure Active Directory only authentication is enabled. +``` + +That message appears whether the address is allowed or not, so a SQL-auth connection test cannot +tell you anything about the firewall on such a server. Test with the Entra login the application +will actually use. + +The third prerequisite cannot be checked until something is deployed, so it is in **Check it worked** +below. + +## The four decisions, and where each one goes + +A good answer names the decision, states the usual right answer, and hands off. + +### 1. What does the application develop against? + +**Usually a real Azure SQL Database.** The free offer runs on General Purpose serverless and is +sized for exactly this. Developing against a different engine and deploying to this one is how +edition, limit and error-semantics differences arrive late. + +- Create it: `provision-azure-sql-db` +- Offline or fast-teardown parity instead: `azuresql-db-scaffold` and `dev-container-templates` +- On serverless, the first connection to a paused database is **expected** to fail while it + resumes: `connect-to-azure-sql` owns that + +### 2. How does the application authenticate? + +**Passwordless, in both places, with one connection string whose only difference is the auth mode.** +The application code does not change between a developer's machine and Azure. What changes is which +credential the identity library discovers. The deployed string Microsoft Learn publishes for the +Python quickstart carries neither a user name nor a password: + +```text +Server=.database.windows.net;Database=;Authentication=ActiveDirectoryMSI;Encrypt=yes;TrustServerCertificate=no; +``` + +**Do not write that keyword from memory for a different driver.** Every stack spells it +differently, one of them has no default mode at all, and a plausible guess produces a string the +driver rejects. `entra-id-auth` carries the accepted value per driver: open it before writing the +auth mode, not after the login fails. + +- Installation, syntax and pooling per stack: `connect-from-dotnet`, `connect-from-python`, + `connect-from-typescript-and-node` +- An error number that needs a cause: `diagnose-connection-errors` + +### 3. Who writes the data layer? + +This is the decision most likely to be answered by reflex, and hand-written CRUD is usually the +wrong reflex. + +| The application needs | Usual answer | Skill | +|---|---|---| +| REST or GraphQL over tables and views that already exist | Generate it. Do not hand-write controllers | `dab-rest-and-graphql` | +| Serverless endpoints, or code that runs when a row changes | Input, output and trigger bindings | `azure-functions-sql-bindings` | +| Rich domain objects and a mapped model | An ORM, chosen for the language | `ef-core-azure-sql`, `prisma-azure-sql`, `sqlalchemy-azure-sql`, `django-azure-sql` | +| A few queries with full control over the SQL | The driver directly | the three `connect-from-*` skills | + +Generated, over a schema that already exists: + +```bash +dab init --database-type mssql --connection-string "@env('SQL_CONNECTION_STRING')" +dab add Book --source dbo.books --source.type table --permissions "authenticated:read" +``` + +Open `dab-rest-and-graphql` before starting that engine rather than after: version 2.0 writes an +`Unauthenticated` provider and serves an MCP endpoint over the same entities, so a short +configuration can publish the whole database anonymously while validation passes. + +Hand-written, on .NET, from an empty directory: + +```bash +dotnet new web -o +cd +dotnet add package Microsoft.Data.SqlClient +``` + +`ef-core-azure-sql` owns Entity Framework Core, including which provider call to configure it with. +For Prisma, pin the major: the connector for this database is a Prisma 7 feature, and an unpinned +install can resolve to a next-major prerelease that has none. + +```bash +npm install --save-dev prisma@7 @prisma/client@7 +./node_modules/.bin/prisma migrate deploy +``` + +Whatever the answer, values reach the database as parameters. `prevent-sql-injection` owns that, and +the Python quickstart carries an explicit warning against shipping its raw statements. + +### 4. How does the schema get there? + +**Through a migration or a schema project, run as its own step.** Not at application startup. + +This needs saying out loud because the sample code contradicts it. Three of the four quickstarts +create their table during startup and label it, in the sample itself, as testing only: "Table should +be created ahead of time in production app". An agent that has absorbed those samples reproduces the +startup DDL and drops the label. + +- The tool-neutral doctrine, including why startup migration is wrong: `schema-migrations-safely` +- Getting the model right before it is expensive to change: `design-azure-sql-schema` + +## Getting it to Azure + +`deploy-app-to-azure` owns the deployment itself. One thing belongs in the plan before it starts. + +**The identity grant is a real step and it is easy to skip.** Connecting a hosted application to the +database is three operations: enable a managed identity, create a database user for it, grant it +roles. One command does all three, and it needs an extension: + +```bash +az extension add --name serviceconnector-passwordless --upgrade + +az webapp connection create sql \ + -g -n \ + --tg --server --database \ + --system-identity --client-type dotnet +``` + +`--client-type` takes one of a closed list that includes `dotnet`, `nodejs`, `python` and `django`. +Behind that one command it also **sets the Microsoft Entra administrator to the signed-in user**, +opens a temporary firewall rule if the local address is blocked and deletes it afterwards, and +writes `AZURE_SQL_CONNECTIONSTRING` onto the application. Changing the administrator changes a +shared server, so say so before running this anywhere but a sandbox. + +**Check what it granted.** The documented grant is `GRANT CONTROL ON DATABASE::"" TO +""`, which Microsoft's own wording invites you to revoke and adjust. The quickstarts grant +`db_datareader` and `db_datawriter` to the same identity instead. `least-privilege-database-roles` +owns what to leave in place. + +## Check it worked + +Three checks, in the order the failures arrive. + +**1. The prerequisites are real, not assumed.** Both commands under Step 0 return a row. An empty +firewall table with a working connection means you are connecting from inside Azure, not that the +rule is unnecessary. The `sqlcmd` round trip returns your Entra identity in `connected_as` and the +user database, never `master`, in `db`. + +**2. The deployed application holds a connection string with no secret in it.** + +```bash +az webapp config appsettings list -g -n \ + --query "[?name=='AZURE_SQL_CONNECTIONSTRING'].value" -o tsv +``` + +Expect one line carrying `Authentication=` and carrying neither `Password=` nor `User ID=`. Empty +output means the connection was never created, which is exactly the failure that presents as a +server error from correct code. + +**3. The identity has a database user, and holds the roles you named.** Run the two queries in +`entra-id-auth`'s own check section against the user database. A `CONTROL` or `db_owner` grant +nobody asked for is the usual leftover of the one-command tool above. + +## Read the source when + +- **A project is about to be generated**: the connect-and-query quickstart for that language on + Microsoft Learn, which is the only current statement of the framework and package list. There is + none for Next.js, and no driver runs in an edge runtime because it speaks a TCP protocol. Django + is supported, through the Microsoft-maintained `mssql-django` backend. +- **A firewall or address question is open**: the Azure SQL Database IP firewall rules article, + which carries the latency table and the auth cache refresh. +- **The identity will not authenticate**: the Microsoft Entra authentication configuration article, + which carries the contained-user requirements and their error codes. +- **The one-command connection is being considered**: the Service Connector passwordless tutorial, + which is the only place the full list of what it changes is written down. + +## Checklist before reporting success + +- [ ] Firewall rule and Microsoft Entra administrator verified, and the latency waited out or flushed +- [ ] One command-line round trip succeeded before any application code was written +- [ ] Connection configuration has no secret, and names the user database +- [ ] The auth mode string came from `entra-id-auth`, not from memory +- [ ] Schema arrives through a migration step, not through startup code +- [ ] The data-layer choice was made against the routing table, not by reflex +- [ ] Deployed identity has a database user with named, justified roles + +## Do not + +- Do not write application code before the firewall rule and the Microsoft Entra administrator are + confirmed. That ordering is the whole skill. +- Do not read a first-run failure as a code or driver bug. Check the latency, then the admin, then + the database user, in that order, and never declare a new firewall rule a success until the + latency is waited out or the auth cache is flushed. +- Do not put a password in configuration because passwordless "can be added later". The two paths + differ in the connection string only, so there is nothing to save. +- Do not copy the quickstart's startup table creation into a real application. The sample says so + itself, in a comment that is easy to drop. +- Do not hand-write CRUD endpoints over an existing schema without first deciding against a + generated data API. +- Do not leave a one-command grant unexamined. `CONTROL` on the database is not an application role. +- Do not restate driver installation, retry policy, pool sizing, Data API builder configuration or + binding syntax here. Every one of those has an owner, and duplicating it is how two skills start + disagreeing. +- Do not answer the local-container version of this question here. That is `azuresql-db-scaffold`. diff --git a/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..28c3723 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/build-app-on-azure-sql/skill.spec.jsonc @@ -0,0 +1,149 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "build-app-on-azure-sql", + "domain": "apis-and-application-patterns", + "value": ["multi-step-with-silent-failures", "corrects-a-wrong-assumption"], + "correction": "Asked to build an application on Azure SQL Database, an agent starts inside the application: it picks a stack, writes a data layer, and puts a SQL user name and password in configuration. Every first-party Azure SQL Database quickstart starts outside the application instead, with three server-side prerequisites that no code can substitute for: a firewall rule covering the client address, a Microsoft Entra administrator on the logical server, and, once the application is deployed, a database user for its managed identity created from a connection that is itself authenticated with Microsoft Entra ID. All three fail silently. The firewall rule takes up to five minutes to take effect, so the first run can fail after the rule was created correctly; the missing Entra admin surfaces as a login failure that reads as a bad credential; and the missing database user surfaces only after deployment, as a server error from an application whose code is correct. The same start-inside-the-code reflex then hand-writes CRUD that Data API builder would generate, and carries the quickstart's create-the-table-at-startup step into a real application, which is the one part of that sample code Microsoft explicitly labels as testing only.", + "value_probe": { + "gate": "The answer puts the server-side prerequisites ahead of the application code, so that by the time the first request runs the client address is admitted, a Microsoft Entra administrator exists on the logical server, and the deployed application's identity has a database user of its own.", + "task": "I have a working Node API on my laptop talking to a local database, and it needs to be running in Azure this week, on App Service, with a new Azure SQL Database behind it. Give me the order of operations, from an empty resource group to the first successful request that reads a row. I would rather not debug this at two in the morning, so be specific about the steps that fail without a clear error, and tell me what I should check after each one.", + "states": [ + "A database user for the application's identity has to be created from a connection that is itself authenticated with Microsoft Entra ID.", + "The first connection can fail after a correct firewall rule was added, because the rule takes several minutes to take effect.", + "A missing Microsoft Entra administrator on the logical server surfaces as a login failure that reads like a bad credential.", + "The work that has to happen outside the application code comes first and no code can substitute for it." + ], + "refutes": [ + "Putting an administrator login and password in the application's configuration is the normal way to connect a deployed application.", + "Granting the application's managed identity a role on the Azure resource is enough, with no step run inside the database itself.", + "Creating the tables from application startup code is a reasonable pattern for a real application.", + "A connection failure straight after adding a firewall rule proves the rule itself is wrong." + ] + }, + "posture": ["read", "write", "provision"], + "maturity": "ga", + "applies_to": ["azure-sql-db"], + "related": [ + "azure-sql", + "provision-azure-sql-db", + "connect-to-azure-sql", + "connect-from-typescript-and-node", + "connect-from-python", + "connect-from-dotnet", + "entra-id-auth", + "diagnose-connection-errors", + "dab-rest-and-graphql", + "azure-functions-sql-bindings", + "ef-core-azure-sql", + "prisma-azure-sql", + "sqlalchemy-azure-sql", + "django-azure-sql", + "design-azure-sql-schema", + "schema-migrations-safely", + "deploy-app-to-azure", + "least-privilege-database-roles", + "prevent-sql-injection", + "azuresql-db-scaffold" + ], + "validation": { + "target": "both", + "setup": [ + "a subscription in which a logical server and database can be created", + "a signed-in Microsoft Entra identity that can be made server administrator", + "an empty project directory, so the ordering is exercised from nothing" + ], + "assert": [ + "the plan verifies the firewall rule and the Microsoft Entra administrator before any application code is written", + "the generated configuration carries no password and no user name, and names the user database rather than master", + "the auth mode written into the configuration is the one the chosen driver actually accepts, and the local and deployed values are stated separately where they differ", + "no table is created during application startup; schema arrives through a migration or a schema project", + "the deployed identity has a database user created from a Microsoft Entra authenticated connection, verifiable in sys.database_principals with authentication_type_desc EXTERNAL", + "a hand-written CRUD layer is proposed only after the generated data API was considered and rejected for a stated reason", + "the roles granted to the application identity are named, and are not left at whatever a one-command tool granted" + ], + "probes": [ + { + "where": "host", + "cmd": "az sql server ad-admin list --help > /dev/null 2>&1", + "expect": { + "succeeds": true + }, + "why": "One of the two read-only commands Step 0 tells a reader to run before writing any application code, and the ordering rule is the whole skill, so a command that is not there is a step that gets skipped. The firewall half of the pair is already probed by diagnose-connection-errors and is deliberately not repeated here. This command has been renamed before in the Azure CLI, which is why remembering it is not good enough. Measured 2026-09-04 on Azure CLI 2.90.0, exit 0. It asserts nothing about a subscription and needs no sign-in." + }, + { + "where": "host", + "cmd": "az webapp connection create sql --help 2>&1 | tr -s '[:space:]' ' ' | grep -q 'Allowed values: django, dotnet, dotnet-internal, go, java, nodejs, none, php, python, ruby, springBoot' && az webapp connection create sql --help 2>&1 | grep -q -- '--system-identity'", + "expect": { + "succeeds": true + }, + "why": "The closed list the skill says --client-type takes, checked verbatim against the command's own help rather than remembered, because the values an agent guesses are the ones this list does not have: node rather than nodejs, csharp or aspnet rather than dotnet. A wrong value here fails the one command that creates the managed identity, the database user and the role grant together, which is the step the skill flags as easy to skip. The whole list is asserted rather than the four values the body names, so an addition is visible too. Measured 2026-09-04 on Azure CLI 2.90.0." + }, + { + "where": "host", + "cmd": "test \"$(az extension list-available --query \"[?name=='serviceconnector-passwordless'] | length(@)\" -o tsv 2>/dev/null)\" = 1", + "expect": { + "succeeds": true + }, + "why": "The extension the skill tells a reader to install by exact name, checked against the live index. az extension add fails on an unknown name, so a renamed or withdrawn extension turns the deployment section's first command into a dead end, and nothing else in the skill can be attempted until it is fixed. Measured 2026-09-04, one match, version 3.3.7. It queries the public index and needs no subscription. Measured by hand on the host on 2026-09-04, exit 0. It is a host probe: nothing in it reaches a database, which is true of all three of the probes above it." + }, + { + "where": "container", + "engine": true, + "cmd": "/opt/mssql-tools18/bin/sqlcmd -S localhost,1433 -U sa -P \"$MSSQL_SA_PASSWORD\" -d probe_build_app_on_azure_sql -C -N -m-1 -Q \"RAISERROR('a severity ten note', 10, 1);\" 2>&1 | grep -q 'Msg 50000'", + "expect": { "succeeds": true }, + "why": "Step 0 tells a reader to run sqlcmd with -m-1 because it makes a severity 10 message print its number, and the body now says that instruction is for the ODBC sqlcmd rather than for go-sqlcmd. This runs the statement through the ODBC build the image ships, at /opt/mssql-tools18/bin/sqlcmd, which is the client the sentence names, so where it runs is half the claim. It ran on the host until 2026-09-05 and was red there, because the host PATH sqlcmd is go-sqlcmd: measured that day on go-sqlcmd 1.10.0 against 18.0.226_4_147, the text is printed and no Msg number is, on stdout or stderr, at -m-1, -m0 and -m1 alike. That reading is what the body was corrected from, and it is kept here because it is the reason this check moved. Measured by hand the same day inside the image on ODBC sqlcmd 18.6.0002.1: Msg 50000, Level 0, State 1 at -m-1, and the bare text without it. mssql-tools18 is deliberately not installed on the host instead, because it would take the sqlcmd name on PATH away from go-sqlcmd and turn the checks in connect-to-azure-sql and diagnose-connection-errors, which read go-sqlcmd's own help output, red. Red if the ODBC build stops printing the number, at which point Step 0 has no build left that does it." + }, + { + "where": "host", + "engine": true, + "cmd": "sqlcmd -S localhost,$PROBE_PORT -U sa -P \"${LAB_SA_PASSWORD:-Str0ng!Lab_Passw0rd_2026}\" -d \"$PROBE_DB\" -C -N -b -Q \"RAISERROR('a severity ten note', 10, 1);\" && ! sqlcmd -S localhost,$PROBE_PORT -U sa -P \"${LAB_SA_PASSWORD:-Str0ng!Lab_Passw0rd_2026}\" -d \"$PROBE_DB\" -C -N -b -Q \"RAISERROR('a real error', 16, 1);\"", + "expect": { "succeeds": true }, + "why": "The other half of the same sentence: -b alone will never turn a severity 10 message into a non-zero exit. Both directions are asserted, because an exit code that is always zero would satisfy the first half on its own and prove nothing. Measured 2026-09-05 on go-sqlcmd 1.10.0: severity 10 exits 0, severity 16 exits 1. Red if -b ever fails on an informational message, which is the case that would make a reader's deployment script stop on a message that is not an error, and red too if it stops failing on a real one." + }, + { + "sql": "SET QUOTED_IDENTIFIER ON; CREATE USER probe_app_identity WITHOUT LOGIN; GRANT CONTROL ON DATABASE::\"probe_build_app_on_azure_sql\" TO \"probe_app_identity\"; SELECT permission_name, state_desc FROM sys.database_permissions WHERE grantee_principal_id = DATABASE_PRINCIPAL_ID('probe_app_identity') AND class = 0 AND permission_name = 'CONTROL';", + "expect": { "rows": [["CONTROL", "GRANT"]] }, + "why": "Check what it granted quotes GRANT CONTROL ON DATABASE::\"\" TO \"\" as the grant the one-command tool leaves behind, and tells a reader to go and look at it. This runs that statement verbatim, on the double-quoted identifier form the documentation uses, and reads the grant back out of sys.database_permissions. The scratch database name is the one the live lane derives from the skill id. Measured 2026-09-05: CONTROL, GRANT. Red if the permission does not land, if it lands as something other than a GRANT of CONTROL at class 0, or if the identifier form is rejected. It is rejected with Msg 102 when QUOTED_IDENTIFIER is OFF, which is how the sqlcmd inside the image connects by default, so the SET is part of the claim rather than decoration." + }, + { + "sql": "DBCC FLUSHAUTHCACHE;", + "expect": { "error": 40518 }, + "why": "Step 0 says DBCC FLUSHAUTHCACHE forces the five minute firewall cache to refresh, does not apply to master, and needs the admin account. None of that can be settled here: measured 2026-09-05, this build refuses the command outright as sa in a user database and in master alike, Msg 40518, DBCC command 'flushauthcache' is not supported in this version of SQL Server. The probe records that boundary as a measurement rather than leaving the claim looking checked. Red if a later build implements the command, which is the day the master-versus-user-database half becomes locally testable." + } + , + { + "sql": "RAISERROR (N'a severity ten note', 10, 1);", + "expect": { + "warning": 50000 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The sqlcmd claim this skill's table turns on, asserted through the ODBC sqlcmd the container image ships, which is the build this lane connects with and the build the -m-1 instructions are for. A severity 10 RAISERROR arrives carrying Msg 50000 rather than as bare text. It goes red if the message stops carrying a number, which is exactly what happens on go-sqlcmd 1.10.0 and is why the table exists. The go-sqlcmd half cannot be asserted here: the lane has one client and it is the ODBC one." + } + ] + }, + "triggering": { + "explicit": ["Use the build-app-on-azure-sql skill"], + "implicit": [ + "build me an app on Azure SQL Database", + "I want to start a new project backed by Azure SQL, where do I begin", + "add Azure SQL Database to this application", + "what is the right way to structure an app against Azure SQL", + "scaffold an API over an Azure SQL database and deploy it" + ], + "routing": [ + "which stack should I use with Azure SQL Database", + "should I write the CRUD endpoints myself or generate them", + "my app works locally but returns a server error once it is deployed to Azure" + ], + "negative": [ + "how do I connect to Azure SQL Database from Node.js", + "add retry logic to my Azure SQL data access layer", + "create an Azure SQL database and open the firewall for my address", + "what does error 40615 mean", + "expose my Orders table as a GraphQL endpoint with Data API builder", + "write a SQL trigger binding for my function app", + "scaffold an app against the local Azure SQL Database container", + "set up Microsoft Entra authentication for my Azure SQL database", + "deploy my existing app to Azure with azd" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet deleted file mode 120000 index 80fd47c..0000000 --- a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet +++ /dev/null @@ -1 +0,0 @@ -../../../skills/connect-from-dotnet \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/SKILL.md b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/SKILL.md new file mode 100644 index 0000000..c7b2d0f --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/SKILL.md @@ -0,0 +1,225 @@ +--- +name: connect-from-dotnet +description: >- + Connects a .NET application to Azure SQL Database with Microsoft.Data.SqlClient: which package + references are required, the encryption defaults and what changed, connection pooling and the + keys that split a pool, and managed identity or other Microsoft Entra ID modes. Use when a user + says "connect .NET to Azure SQL", "Microsoft.Data.SqlClient", "SqlConnection", "passwordless + access for my .NET app", "managed identity for my API", "Active Directory Default", "Max Pool + Size", "Encrypt=Strict", or asks whether "Microsoft.Data.SqlClient.Extensions.Azure" is needed. + Also use when an app authenticates with a password locally and must use an identity in Azure. + Covers packages, connection string keywords, pooling and Entra ID for .NET only; the ORM path is + ef-core-azure-sql, and encryption doctrine plus transient-fault retry belong to + connect-to-azure-sql. +--- + +# Connect from .NET + +The driver is `Microsoft.Data.SqlClient`, encryption is on by default, pooling is automatic, and +since version 7.0 Microsoft Entra authentication needs a **second package reference** that nothing +in the connection string hints at. + +Measured 2026-09-03 on `Microsoft.Data.SqlClient` 7.0.2, `Microsoft.Data.SqlClient.Extensions.Azure` +7.0.2 and .NET SDK 8.0.421, with the commands in **Check it worked**. + +Encryption doctrine, retry and pool sizing are the same in every language and live in +`connect-to-azure-sql`. Entity Framework Core is `ef-core-azure-sql`. + +## The packages + +```bash +dotnet add package Microsoft.Data.SqlClient +``` + +`System.Data.SqlClient` is the older provider and receives no new features. A +`using System.Data.SqlClient;` in a new file is a mistake worth fixing first. + +## Gotcha: Entra ID needs a second package from version 7.0 + +7.0 moved the Entra dependencies out of the core package. Read the restored graph, do not guess: + +```bash +dotnet list package --include-transitive | grep -E '> (Azure\.Identity|Azure\.Core|Microsoft\.Identity\.Client) ' +``` + +On the core package alone that prints nothing, so the driver registers no Entra provider, and you +can see that without a server at all: + +```csharp +using Microsoft.Data.SqlClient; +var provider = SqlAuthenticationProvider.GetProvider(SqlAuthenticationMethod.ActiveDirectoryDefault); +Console.WriteLine(provider is null ? "NONE" : provider.GetType().FullName); +``` + +`NONE` means every `Active Directory *` mode has nothing to resolve to. Against a server that offers +Microsoft Entra authentication the login then raises `ArgumentException`: `Cannot find an +authentication provider for 'ActiveDirectoryDefault'`, naming the package to install. The fix is +that package reference and no code change: + +```bash +dotnet add package Microsoft.Data.SqlClient.Extensions.Azure +``` + +Re-run both commands. The graph now lists `Azure.Identity`, `Azure.Core` and +`Microsoft.Identity.Client`, and the provider prints +`Microsoft.Data.SqlClient.ActiveDirectoryAuthenticationProvider`. The extension registers itself. +SQL authentication, and applications that supply their own token through `AccessToken` or +`AccessTokenCallback`, keep the lighter graph. + +**The local container is not where that exception reproduces.** The provider is looked up during +the login exchange, so the `ArgumentException` needs a server offering Microsoft Entra +authentication. Against the Azure SQL Database container, which has no Entra configuration, +`Authentication=Active Directory Default` fails earlier and differently, as `Msg 18456`, +`Login failed for user ''`, naming neither a provider nor a package; that one is +`diagnose-connection-errors`. Decide the package from the graph and the mode, never from whether a +local connection happened to succeed. + +## Encryption defaults, and the version that changed them + +`Encrypt` defaults to `True` from version 4.0 and defaulted to `False` in 3.x and earlier. +`TrustServerCertificate` defaults to `False`, and under `Encrypt=Strict` it is ignored and treated +as false. So encryption is on and the certificate is validated, and connecting forces a trust +decision rather than deferring one. Code written against a 3.x default surfaces that as a +connection error on upgrade rather than as an upgrade note. Reach for `connect-to-azure-sql` before +changing either value; the answer is almost never `TrustServerCertificate=True`. + +## Pooling is automatic, and the connection string is the key + +Never cache a `SqlConnection`. Open one, use it, dispose it. The pool underneath does the reuse. + +```csharp +await using var connection = new SqlConnection(connectionString); +await connection.OpenAsync(cancellationToken); +``` + +Four things decide whether pooling actually helps, with every default below read out of +`SqlConnectionStringBuilder` on 7.0.2: + +- **The pool key is the whole connection string.** Two strings differing in any way, including a + per-tenant `Application Name` or a rearranged keyword, get two pools that both count against the + same server limits. Build the string once. +- **`Connect Timeout` defaults to 15 seconds**, and Microsoft's guidance for Azure SQL Database is + 30. Set it deliberately. `Pooling` is `True`, `Max Pool Size` 100 and `Min Pool Size` 0. +- **`Connect Retry Count`, 1, and `Connect Retry Interval`, 10, are idle-connection resiliency + inside the driver.** It transparently reconnects a dropped idle connection; it does not replay a + failed query. From version 5 the effective default is raised for Azure SQL Database endpoints and + again for serverless ones, so read it from the documentation rather than from the builder. + Application-level retry is separate, and is `connect-to-azure-sql`. Keep both. +- **`AccessTokenCallback` is part of the pool key.** A new lambda per `SqlConnection` creates a new + pool per connection. Reference one shared static callback. + +Sizing the pool, and why the limit that bites is workers rather than sessions, is +`connect-to-azure-sql`. + +## Microsoft Entra ID + +Set `Authentication` in the connection string. `Active Directory Managed Identity` for an +Azure-hosted workload, which is the production mode; `Active Directory Default` for development +machines and CI, where the credential varies; `Active Directory Service Principal` for a registered +app, client id in `User Id` and secret in `Password`. `Active Directory Password` is deprecated and +obsolete in 7.0. + +System-assigned managed identity needs nothing beyond the mode. User-assigned supplies the identity +in `User Id`, and **which identifier** depends on the driver version: the **client id** from 3.0 +onwards, the **object id** on 2.1. Everything current wants the client id, and the object id fails +at token acquisition with an error pointing at the identity endpoint, not at the connection string. + +```csharp +var connectionString = + "Server=tcp:.database.windows.net,1433;" + + "Database=;" + + "Authentication=Active Directory Managed Identity;" + + "User Id=;" + // omit this line for a system-assigned identity + "Encrypt=Mandatory;Connect Timeout=30;"; +``` + +`Active Directory Default` walks a chain of credential providers on the first connection, failing +ones first, so it costs latency a production workload has no reason to pay. Name the credential type +directly outside development. + +The identity still needs a database principal before it can connect. Creating it is `entra-id-auth`; +wiring the identity across Azure resources is `managed-identity-across-azure-services`. + +## Check it worked + +Four checks. The first three need no database, so run them before blaming the network. + +**One: the dependency graph matches the authentication mode.** From the project directory: + +```bash +dotnet list package --include-transitive | grep -cE '> (Azure\.Identity|Azure\.Core|Microsoft\.Identity\.Client) ' +``` + +Expect `3` on a project that uses any `Active Directory *` mode, and `0` on one that does not. +Measured 2026-09-03 on 7.0.2 with the extension package: `Azure.Identity` 1.18.0, `Azure.Core` +1.51.1, `Microsoft.Identity.Client` 4.84.2. Without it, all three absent. + +**Two: the driver has a provider for the mode you set.** Run the `GetProvider` snippet above. Expect +`Microsoft.Data.SqlClient.ActiveDirectoryAuthenticationProvider`; `NONE` means the extension package +is missing. That answer is the same on a laptop as in production. + +**Three: nothing in the tree undoes the defaults.** Both expect no output: + +```bash +grep -rn 'System\.Data\.SqlClient' --include='*.cs' . +grep -rniE 'TrustServerCertificate *= *true' --include='*.cs' --include='*.json' --include='*.config' . +``` + +**Four: the connection you opened is the one you meant.** Ask the engine, over the application's own +connection rather than from a query tool signed in as somebody else: + +```csharp +await using var connection = new SqlConnection(connectionString); +await connection.OpenAsync(); +await using var cmd = new SqlCommand( + "SELECT c.encrypt_option, SUSER_SNAME() AS login, USER_NAME() AS db_user, DB_NAME() AS db " + + "FROM sys.dm_exec_connections AS c WHERE c.session_id = @@SPID;", connection); +await using var r = await cmd.ExecuteReaderAsync(); +if (await r.ReadAsync()) Console.WriteLine($"encrypt={r[0]} login={r[1]} user={r[2]} db={r[3]}"); +``` + +One row, and each column settles a different question. `encrypt_option` is `TRUE` on an encrypted +connection, so anything else means an `Encrypt=False` survived somewhere in the string. `login` is +the identity the server believes it is talking to: for managed identity the Entra principal, and if +it is the SQL login you meant to stop using, an old connection string is still in play. `db` catches +the connection that opened happily against the wrong database. + +**Zero rows means a permission problem, not an unencrypted connection.** `sys.dm_exec_connections` +needs `VIEW DATABASE STATE` on the database, or membership in the `##MS_ServerStateReader##` server +role, and a least-privileged application user has neither by default. There is no unprivileged +substitute: `CONNECTIONPROPERTY` carries `net_transport`, `protocol_type` and `auth_scheme`, not +`encrypt_option`, and returns NULL for any other property name. + +## Do not + +- Do not assume a connection string alone enables Entra ID on 7.0 or later. It needs the extension + package, and `GetProvider` says in one line whether it is there. +- Do not start new work on `System.Data.SqlClient`. +- Do not set `TrustServerCertificate=True` to clear a certificate error. Read `connect-to-azure-sql`. +- Do not cache or share a `SqlConnection` across requests, and do not vary the connection string per + request, per tenant or per call site without meaning a separate pool. Cache nothing; the pool is + the cache. +- Do not use `Active Directory Password`, and do not pass a managed identity object id to a 3.0 or + later driver; it wants the client id. +- Do not hard-code a password, token, server hostname or identity id. +- Do not write retry loops here. Transient-fault handling is one policy for every language, in + `connect-to-azure-sql`. + +## References + +- Open `references/entra-modes-and-connection-keywords.md` when the mode you need is not one of the + three above, or before setting a keyword this body does not list: every `Authentication` value + with the driver version it arrived in, and the encryption and pooling defaults read off 7.0.2. +- [Entra authentication with + SqlClient](https://learn.microsoft.com/sql/connect/ado-net/sql/azure-active-directory-authentication): + every mode, the version it arrived in, and the 7.0 migration steps. Read before choosing a mode or + debugging a provider error. +- [SqlConnection.ConnectionString](https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring): + every keyword with its default. Read it rather than recalling a default. +- [SQL Server connection pooling](https://learn.microsoft.com/sql/connect/ado-net/sql-server-connection-pooling): + how the pool key is formed. Read it when two code paths appear to fight over connections. +- [sys.dm_exec_connections](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-connections-transact-sql): + the columns check four reads and the permission it needs. Read it when that query returns no row. +- `connect-to-azure-sql`: encryption doctrine, retry, pool sizing, and the first-connect error on a + paused database. `ef-core-azure-sql`: the same database through an object-relational mapper, whose + connection string still comes from here. diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/references/entra-modes-and-connection-keywords.md b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/references/entra-modes-and-connection-keywords.md new file mode 100644 index 0000000..d9e4773 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/references/entra-modes-and-connection-keywords.md @@ -0,0 +1,57 @@ +# Microsoft Entra ID modes and connection string keywords + +`Microsoft.Data.SqlClient` reference detail. SKILL.md carries the three modes most +applications need and the defaults that change behaviour; this file carries the full lists. + +- [Authentication modes](#authentication-modes) +- [Encryption keywords](#encryption-keywords) +- [Pooling and connection keywords](#pooling-and-connection-keywords) + +## Authentication modes + +Every value accepted by the `Authentication` connection string keyword, with the driver version it +arrived in. All of them need the `Microsoft.Data.SqlClient.Extensions.Azure` package reference on +7.0 and later. + +| Value | From | Use it for | +|---|---|---| +| `Active Directory Managed Identity` or `Active Directory MSI` | 2.1.0 | An Azure-hosted workload. The production mode | +| `Active Directory Default` | 3.0.0 | Development machines and CI, where the credential varies | +| `Active Directory Workload Identity` | 5.2.0 | A federated identity on a Kubernetes-style host | +| `Active Directory Service Principal` | 2.0.0 | A registered app, client id in `User Id`, secret in `Password` | +| `Active Directory Interactive` | 2.0.0 | A person at a browser, with multifactor authentication | +| `Active Directory Device Code Flow` | 2.1.0 | A host with no browser. Raise `Connect Timeout` for the sign-in | +| `Active Directory Integrated` | 2.0.0 | A domain-joined Windows client | +| `Active Directory Password` | 1.0 | Nothing. Deprecated, and obsolete in 7.0 | + +`Active Directory Default` walks a chain of credential providers on every first connection, failing +ones first, so it costs latency a production workload has no reason to pay. + +## Encryption keywords + +| Keyword | Default | Notes | +|---|---|---| +| `Encrypt` | `True` from version 4.0. `False` in 3.x and earlier | Values in 5.0 and later: `true`/`mandatory`, `false`/`optional`, and `strict` for TDS 8.0 | +| `TrustServerCertificate` | `False` | With `Encrypt=Strict` it is ignored and treated as false | +| `HostNameInCertificate` | not set | Available from 5.0, for when the certificate name differs from the data source | + +## Pooling and connection keywords + +Read out of `SqlConnectionStringBuilder` on 7.0.2, measured 2026-09-03 on .NET SDK 8.0.421. + +| Keyword | Default | +|---|---| +| `Pooling` | `True` | +| `Max Pool Size` | 100 | +| `Min Pool Size` | 0 | +| `Connect Timeout` | 15 | +| `Load Balance Timeout` | 0 | +| `Multiple Active Result Sets` | `False` | +| `Connect Retry Count` | 1 | +| `Connect Retry Interval` | 10 | + +`Connect Retry Count` and `Connect Retry Interval` are idle-connection resiliency inside the driver. +The builder reports the library default; from version 5 the effective default is raised for Azure SQL +Database endpoints and again for serverless ones, so read the current number from +[SqlConnection.ConnectionString](https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring) +rather than from the builder. diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/skill.spec.jsonc new file mode 100644 index 0000000..31c5514 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-dotnet/skill.spec.jsonc @@ -0,0 +1,126 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "connect-from-dotnet", + "domain": "drivers-and-connectivity", + "value": ["not-in-training-data", "corrects-a-wrong-assumption"], + "correction": "Agents treat Microsoft Entra authentication in .NET as a connection string setting alone, writing Authentication=Active Directory Default against a single Microsoft.Data.SqlClient package reference; since version 7.0 the core package no longer carries the Entra dependencies, confirmed in the restored graph for 7.0.2, so it registers zero authentication providers and every Active Directory mode has nothing to resolve to until Microsoft.Data.SqlClient.Extensions.Azure is added as a second package reference. Which failure a developer sees depends on the server, because the provider is looked up during the login exchange rather than before it: a server offering Microsoft Entra authentication produces ArgumentException, Cannot find an authentication provider, at Open time, while the local Azure SQL Database container refuses the login first and returns SqlException 18456, measured on 2026-08-28, which is a subject for diagnose-connection-errors rather than for this skill. The package rule is a property of the dependency graph and holds on both, so a local run is not where the exception reproduces and is not evidence that the extension package can be skipped.", + "value_probe": { + "task": "Our .NET 8 background worker talks to a database in Azure using Microsoft.Data.SqlClient directly. There is no ORM, we open a SqlConnection and run commands. The project file has exactly one database package in it:\n\n```xml\n\n \n\n```\n\nSecurity asked us to stop shipping a password, so we changed the connection string from a user name and password to this:\n\n```\nServer=tcp:ourserver.database.windows.net,1433;Database=orders;Authentication=Active Directory Default;Encrypt=True;\n```\n\nIt compiles. It restores. The moment the worker calls OpenAsync in Azure it throws before anything reaches the database, and the exception is complaining that it cannot find something it needs rather than reporting a login failure. Reverting the connection string to the user name and password makes it work again, so the server and the network are fine.\n\nAn engineer has spent two days checking that the worker's identity exists and has been granted access on the database side. What is actually wrong, and what do we change?", + "gate": "The answer identifies that the project is missing a package the driver needs for the Active Directory authentication modes to resolve to anything, and adds that second package reference, rather than only editing the connection string or making further changes on the database and identity side.", + "states": [ + "The base driver package on its own registers no authentication provider for the Active Directory modes, so the connection string value has nothing to resolve to.", + "The fix is a second package reference added to the project, not a change to the connection string.", + "The provider is resolved during the login exchange, which is why the failure appears at connection open time and not at startup or at build time.", + "The identity work already done on the database side is necessary but is not what is failing here." + ], + "refutes": [ + "A single reference to the base data provider package is all that is needed for the Active Directory authentication modes to work, and no additional package is required.", + "The failure means the application identity has not been created as a database user or has not been granted the right database role.", + "The connection string keyword is spelled wrong, or a different Active Directory mode value should be substituted to make it work.", + "Downgrading or upgrading the single existing package to a different version restores the Active Directory modes without adding anything else." + ] + }, + "posture": ["read", "write", "execute"], + // Demoted from preview to draft on 2026-09-20, and the reason is a direction + // rather than a verdict. The value measurement of 2026-09-05, ten runs per arm, + // returned UNCLEAR, and inside it the skill makes the answer worse twice. The + // statement that the provider is resolved during the login exchange reads 1 of 10 + // with the skill against 6 of 10 without, so loading the skill costs a fact the + // model already had. A falsehood, that the connection string keyword is spelled + // wrong or wants a different Active Directory value, reads 4 of 10 with against 2 + // of 10 without, so loading it also nudges the reader toward the one edit that + // cannot fix this. The gate is 10 of 10 in both arms. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-to-azure-sql", + "diagnose-connection-errors", + "ef-core-azure-sql", + "dapper-and-ado-net-azure-sql", + "managed-identity-across-azure-services" + ], + "validation": { + "target": "both", + "setup": [ + "a .NET project referencing Microsoft.Data.SqlClient 7.0 or later", + "a reachable Azure SQL database, or the local container" + ], + "assert": [ + "a project using any Active Directory authentication mode also references Microsoft.Data.SqlClient.Extensions.Azure", + "the project references Microsoft.Data.SqlClient and not System.Data.SqlClient", + "Encrypt is absent or set to Mandatory or Strict, and TrustServerCertificate is absent or false", + "a user-assigned managed identity supplies its client id in User Id", + "connection strings differ only where a separate pool is intended", + "the extension package requirement is decided from the dependency graph and the authentication mode, not from the outcome of a connection to the local container" + ], + "probes": [ + { + "cmd": "dotnet list package --help | grep -q -- '--include-transitive'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The flag both dependency-graph checks in SKILL.md depend on. Read out of the tool's own help on .NET SDK 8.0.421, measured 2026-09-03. dotnet is a client SDK a developer installs, never something the engine image ships, so this is a host probe and it opens no database connection." + }, + { + "cmd": "d=$(mktemp -d) && cd \"$d\" && dotnet new console -o app >/dev/null && cd app && dotnet add package Microsoft.Data.SqlClient >/dev/null && before=$(dotnet list package --include-transitive | grep -cE '> (Azure\\.Identity|Azure\\.Core|Microsoft\\.Identity\\.Client) ' || true) && dotnet add package Microsoft.Data.SqlClient.Extensions.Azure >/dev/null && after=$(dotnet list package --include-transitive | grep -cE '> (Azure\\.Identity|Azure\\.Core|Microsoft\\.Identity\\.Client) ' || true) && test \"$before\" = 0 && test \"$after\" = 3", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The correction this skill exists for, made executable: none of Azure.Identity, Azure.Core or Microsoft.Identity.Client is in the restored graph of Microsoft.Data.SqlClient alone, and all three appear once Microsoft.Data.SqlClient.Extensions.Azure is referenced. Measured 2026-09-03 on 7.0.2 with .NET SDK 8.0.421: 0 then 3, with Azure.Identity 1.18.0, Azure.Core 1.51.1 and Microsoft.Identity.Client 4.84.2. A host probe, and it needs NuGet access to restore the two packages into a throwaway project." + }, + { + "sql": "SELECT COUNT(*) AS rows_for_this_session FROM sys.dm_exec_connections WHERE session_id = @@SPID;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The fourth check in SKILL.md reads encrypt_option, SUSER_SNAME, USER_NAME and DB_NAME over the application's own connection, and all of it rests on sys.dm_exec_connections returning the row for the current session. One row here means the check is runnable; zero means the connection lacks VIEW DATABASE STATE, which is the permission case the section calls out." + }, + { + "sql": "SELECT CASE WHEN CONNECTIONPROPERTY('auth_scheme') IS NOT NULL AND CONNECTIONPROPERTY('encrypt_option') IS NULL THEN 1 ELSE 0 END AS no_unprivileged_substitute;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "SKILL.md tells a reader there is no unprivileged substitute for the DMV, because CONNECTIONPROPERTY carries auth_scheme but not encrypt_option and returns NULL for any other property name. Sourced from CONNECTIONPROPERTY (Transact-SQL) on Microsoft Learn, whose argument table lists seven property names, none of them encrypt_option, and ends with 'Any other string: Returns NULL for invalid input'." + } + ] + }, + "triggering": { + "explicit": [ + "Use the connect-from-dotnet skill", + "Connect my .NET app to Azure SQL Database", + "Set up managed identity authentication with Microsoft.Data.SqlClient" + ], + "implicit": [ + "do I need Microsoft.Data.SqlClient.Extensions.Azure to use Active Directory Default", + "my API worked locally with a password but fails in Azure with a login error", + "should I use System.Data.SqlClient or Microsoft.Data.SqlClient", + "how do I size the connection pool for a .NET service on Azure SQL" + ], + "routing": [ + "add a data access layer to my ASP.NET service", + "passwordless database access for a .NET app running in Azure", + "add a data access layer to my ASP.NET service that talks to Azure SQL" + ], + "negative": [ + "connect to Azure SQL from Python with mssql-python", + "set the mssql pool size in my Node.js API", + "add an EF Core migration for a new column", + "why does my first connection to a paused database fail" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-python b/plugins/microsoft-sql-vscode/skills/connect-from-python deleted file mode 120000 index 95a1bba..0000000 --- a/plugins/microsoft-sql-vscode/skills/connect-from-python +++ /dev/null @@ -1 +0,0 @@ -../../../skills/connect-from-python \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-python/SKILL.md b/plugins/microsoft-sql-vscode/skills/connect-from-python/SKILL.md new file mode 100644 index 0000000..68c04be --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-python/SKILL.md @@ -0,0 +1,265 @@ +--- +name: connect-from-python +description: >- + Connects a Python application to Azure SQL Database, choosing between Microsoft's first-party + mssql-python driver and the incumbent pyodbc, and covering driver installation, the connection + string each one wants, connection pooling, and Microsoft Entra ID including token-based + authentication. Use when a user says "connect Python to Azure SQL", "mssql-python", "pyodbc", + "install the ODBC driver so my Python code can connect", "which Python driver for SQL Server", + or hits errors such as "data source name not found and no default driver specified" or a + container image that cannot load the driver. Also use when adding a database layer to a Python + service, or when it needs a passwordless connection. Covers installation, connection strings, + pooling and Entra ID for Python only. Encryption doctrine and transient-fault retry belong to + connect-to-azure-sql; Node and .NET have their own skills. +--- +# Connect from Python + +The Python answer changed. There are now two supported drivers, and the newer one removes the step +that causes most Python connection failures: installing an ODBC driver into the operating system. + +Checked on 2026-09-03 against pyodbc 5.3.0, msodbcsql18 18.6.2.1 and unixODBC 2.3.14 on this +machine, and the current documentation for mssql-python 1.13.0. + +This skill owns driver choice, installation, connection string shape, pooling and Entra ID for +Python. Encryption doctrine, retry and the first-connect error on a paused database are the same in +every language and live in `connect-to-azure-sql`. SQLAlchemy is `sqlalchemy-azure-sql`. + +## Choose the driver first + +| | `mssql-python` 1.13.0 | `pyodbc` 5.3.0 | +|---|---|---| +| Who ships it | Microsoft, first-party | Community, long-standing | +| Python required | 3.10 or later | 3.9 or later | +| Operating system install | **None.** `mssql-python-odbc` carries the driver binaries | `msodbcsql18` and a driver manager, per distribution | +| Entra `ActiveDirectoryDefault` | Yes, and `azure-identity` comes with it | No, see below | + +**Use `mssql-python` for new code.** Skipping the operating system install removes a class of +container and CI failures. **Keep `pyodbc` when something above it already requires it**, most often +an ORM dialect written against it: `sqlalchemy-azure-sql`, `django-azure-sql`. Do not rewrite a data +access layer just to change driver. + +```bash +pip install mssql-python # first-party path +pip install pyodbc # incumbent path, plus the driver install below +``` + +On Python 3.9 the first line fails with `No matching distribution found`, which reads like a network +problem and is not one. Measured here on 3.9.6. + +## Installing the ODBC driver, for the pyodbc path only + +The package is **`msodbcsql18`**, the connection string names `ODBC Driver 18 for SQL Server`, and +the two have to agree exactly. `pip install pyodbc` succeeds without it and the failure arrives at +the first connect, which is why the first check below exists. + +Three things break an unattended build: the licence prompt, Alpine having no Microsoft package +repository, and slim Debian omitting `libgssapi-krb5-2`. Open +[references/odbc-driver-install-on-linux.md](references/odbc-driver-install-on-linux.md) before you +write a Dockerfile or CI step that installs the driver. + +## Connection strings + +Both take ODBC-style keywords. `mssql_python.connect()` takes the same string with `Driver` left +out, because the driver is inside the package. `pyodbc` needs it named, and the name is the one the +install registered: + +```python +import pyodbc + +conn = pyodbc.connect( + "Driver={ODBC Driver 18 for SQL Server};" + f"Server=tcp:{server},1433;" + f"Database={database};" + "Encrypt=yes;" +) +``` + +Read the server, database and any credential from the environment or a secret store, never a +literal in source, and bind every value in a query rather than formatting it into the text. + +## Pooling + +**`mssql-python` pools by default**, `max_size` 100 and `idle_timeout` 600 seconds. + +```python +import mssql_python + +# Call this ONCE, before the first connect. Afterwards it has no effect. +mssql_python.pooling(max_size=25, idle_timeout=300) +``` + +A `pooling()` call after any connect is silently ignored, which looks exactly like a setting that +did not take. The pool key is the connection string byte for byte, so one extra keyword or a +different capitalisation on one call site produces a second independent pool: build it once as a +constant. Under managed identity, device code or a `token_provider` the identity joins the key. +There is no `ClearPool`, no pool statistics and no minimum size. + +**`pyodbc` does not pool in Python.** Pooling happens in the ODBC layer, `pyodbc.pooling` reads +`True` out of the box, and it is switched off by assigning `False` **before any connection is +made**. Its context manager is not `mssql-python`'s: `Connection.__exit__` commits, or rolls back on +an exception, and **does not close**, so `with pyodbc.connect(...)` alone never returns the +connection. Call `close()` in a `finally`. Never enable `pyodbc.pooling` and a pool above it at +once. + +Pool sizing, and why the binding limit is workers rather than sessions, is `connect-to-azure-sql`. + +## Microsoft Entra ID + +### With mssql-python + +Set the `Authentication` keyword: `ActiveDirectoryDefault` locally, because it picks up a CLI +sign-in, and `ActiveDirectoryMSI` for a managed identity in production. + +```python +conn = mssql_python.connect( + f"Server=tcp:{server},1433;Database={database};" + "Authentication=ActiveDirectoryMSI;" + f"UID={client_id};" # only for a user-assigned managed identity + "Encrypt=yes;" +) +``` + +`ActiveDirectoryDefault` walks a chain of providers on the first connection, failures first, so it +costs seconds production has no reason to pay. Name the credential type directly there. Where the +credential is an `azure-identity` object, `token_provider=` takes it whole and refreshes tokens for +pooled connections. + +### With pyodbc + +**The ODBC driver has no `ActiveDirectoryDefault`.** It is not a slower path, it is a rejected +string, and the driver says so before it opens a socket: + +```bash +python3 -c " +import pyodbc +try: + pyodbc.connect('Driver={ODBC Driver 18 for SQL Server};Server=tcp:127.0.0.1,1;' + 'Authentication=ActiveDirectoryDefault;Encrypt=yes;', timeout=2) +except pyodbc.Error as e: + print(e.args[0], e.args[1]) +" +``` + +Measured 2026-09-03: `08001`, `Invalid value specified for connection string attribute +'Authentication'`. Swap in `ActiveDirectoryMsi` and the same command reaches `HYT00`, login timeout, +because that value is accepted. The keyword takes `SqlPassword`, `ActiveDirectoryIntegrated`, +`ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, `ActiveDirectoryServicePrincipal` and the +deprecated `ActiveDirectoryPassword`, and nothing else. + +To get the same behaviour, acquire the token in Python and hand it to the driver: + +```python +import struct +from azure.identity import DefaultAzureCredential +import pyodbc + +# msodbcsql.h: SQL_COPT_SS_BASE_EX (1240) + 16 +SQL_COPT_SS_ACCESS_TOKEN = 1256 + +token = DefaultAzureCredential().get_token("https://database.windows.net/.default").token +packed = token.encode("utf-16-le") +token_struct = struct.pack(f"' : file not +found`. `IM002`, `Data source name not found and no default driver specified`, is a string that +named no driver at all. Installing `msodbcsql18` answers the first and nothing about the second. + +**Two: encrypted, as the principal you meant, against the database you meant.** Save this as +`check_connection.py`: + +```python +import os, sys, pyodbc + +CONN = ( + "Driver={ODBC Driver 18 for SQL Server};" + f"Server=tcp:{os.environ['SQL_SERVER']},1433;Database={os.environ['SQL_DATABASE']};" + f"UID={os.environ['SQL_USER']};PWD={os.environ['SQL_PASSWORD']};" + "Encrypt=yes;TrustServerCertificate=no;" +) + +conn = pyodbc.connect(CONN, timeout=15) +try: + db, login, user, enc = conn.cursor().execute( + "SELECT DB_NAME(), SUSER_SNAME(), USER_NAME()," + " (SELECT encrypt_option FROM sys.dm_exec_connections WHERE session_id = @@SPID)" + ).fetchone() +finally: + conn.close() + +print(f"database={db} login={login} user={user} encrypted={enc}") +sys.exit(0 if (enc == "TRUE" and db == os.environ["SQL_DATABASE"] + and login == os.environ["SQL_EXPECT_LOGIN"]) else 1) +``` + +```bash +export SQL_SERVER=.database.windows.net SQL_DATABASE= \ + SQL_USER= SQL_PASSWORD= SQL_EXPECT_LOGIN= +python3 check_connection.py; echo "exit $?" +``` + +Expect exit `0` and a line ending `encrypted=TRUE`. Read the fields, not only the exit code: + +- `encrypted=FALSE` is driver 17, whose `Encrypt` default is `no`. Driver 18 defaults to `yes`. +- `login` is who authenticated. A credential chain that picked up your own sign-in instead of the + managed identity shows up here and nowhere else. +- `user` is the database principal that login mapped to. Anything but the one you granted means the + grant landed in a different database. +- `TrustServerCertificate=yes` still prints `TRUE`: it keeps encryption and drops validation. Azure + SQL Database needs no such exemption; the container's self-signed certificate does, and + `connect-to-azure-sql` owns which. +- A permissions error from `sys.dm_exec_connections` is `VIEW DATABASE STATE`, not encryption: on + Basic, S0, S1 and elastic pool databases only an administrator can read it. + +## Do not + +- Do not assume `pyodbc` is the only Python driver. That was true and is not. +- Do not add an operating system ODBC driver install to a `mssql-python` deployment. +- Do not put `Authentication=ActiveDirectoryDefault` in a `pyodbc` string, or an access token + alongside `UID`, `PWD` or `Authentication`. +- Do not use `ActiveDirectoryPassword`, or `ActiveDirectoryDefault` in production. +- Do not build SQL text with string formatting. Bind parameters. +- Do not write retry loops here. Transient-fault handling is one policy for every language, in + `connect-to-azure-sql`. + +## References + +- [Microsoft Python driver for SQL Server](https://learn.microsoft.com/sql/connect/python/mssql-python/python-sql-driver-mssql-python): + read it before writing new Python data access. +- [Microsoft Entra authentication with mssql-python](https://learn.microsoft.com/sql/connect/python/mssql-python/entra-authentication): + read it when choosing a mode, for the authoritative list and `token_provider`. +- [Using Microsoft Entra ID with the ODBC driver](https://learn.microsoft.com/sql/connect/odbc/using-azure-active-directory): + read it before claiming the driver supports a mode. Carries the access token structure. +- [DSN and connection string keywords](https://learn.microsoft.com/sql/connect/odbc/dsn-connection-string-attribute): + read it before changing an encryption or certificate keyword. `Encrypt` defaults to `yes` from + version 18. +- `connect-to-azure-sql`: encryption doctrine, retry, pool sizing, and the first-connect error on a + paused database. diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-python/references/odbc-driver-install-on-linux.md b/plugins/microsoft-sql-vscode/skills/connect-from-python/references/odbc-driver-install-on-linux.md new file mode 100644 index 0000000..dd4cbca --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-python/references/odbc-driver-install-on-linux.md @@ -0,0 +1,57 @@ +# Installing msodbcsql18 on Linux, for the pyodbc path + +Only the `pyodbc` path needs this. `mssql-python` carries the driver binaries in its own wheel and +an operating system install added alongside it is dead weight. + +`pip install pyodbc` never needs a compiler: the project publishes both `manylinux` and +`musllinux` wheels, so a musl based image builds as cleanly as a glibc one. What is missing at +runtime is the driver the connection string names. + +## The name has to match, exactly + +The package is `msodbcsql18`. The connection string keyword is +`Driver={ODBC Driver 18 for SQL Server}`. A registered driver under any other name is not found, +and the failure is `01000`, `Can't open lib '' : file not found`, not `IM002`. + +Confirm what the driver manager actually registered: + +```bash +python3 -c "import pyodbc; print(pyodbc.drivers())" +odbcinst -q -d +``` + +## Three things that break an unattended build + +**The licence prompt.** The install is non interactive only if the licence is accepted explicitly. +Either set `ACCEPT_EULA=Y` on the install command, or, from driver 18.4, create the file +`/opt/microsoft/msodbcsql18/ACCEPT_EULA` before installing. Without one of the two, a container +build hangs on a prompt nobody can answer and eventually times out with no useful message. + +**Alpine has no Microsoft package repository.** Every other supported distribution installs from +`packages.microsoft.com`. On Alpine the `.apk` is downloaded and installed with +`apk add --allow-untrusted`, and the signature check is skipped because there is no repository key +to check it against. + +**Slim Debian and Ubuntu images omit `libgssapi-krb5-2`.** The driver links against it. Install it +alongside `msodbcsql18` or the driver loads and then fails at connect time. + +## Do not recite the per distribution commands + +They change with releases, and a stale `apt` source line for the wrong Debian version is one of the +more common ways this fails. Fetch the current ones: + +[Install the Microsoft ODBC driver for SQL Server on Linux](https://learn.microsoft.com/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server) + +## Checking a built image + +Run this as the last step of the build, not the first step of the application: + +```bash +docker run --rm python3 -c "import pyodbc, sys; \ + ok = 'ODBC Driver 18 for SQL Server' in pyodbc.drivers(); \ + print(pyodbc.drivers()); sys.exit(0 if ok else 1)" +``` + +Exit `0` means the driver is present and named as the connection string expects. Exit `1` means +`pip install pyodbc` succeeded and the operating system install did not, which is the same state +that produces `01000` at the first connect and gets misread as a network problem. diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-python/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/connect-from-python/skill.spec.jsonc new file mode 100644 index 0000000..7ff7bc5 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-python/skill.spec.jsonc @@ -0,0 +1,132 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "connect-from-python", + "domain": "drivers-and-connectivity", + "value": ["not-in-training-data", "changes-too-fast-to-memorize"], + "correction": "Asked to connect Python to Azure SQL Database, an agent reaches for pyodbc and a platform-specific ODBC driver install, because for years that was the only answer; Microsoft now ships a first-party driver, mssql-python, that carries the driver binaries inside its own wheel so there is nothing to install at the operating system level, and that accepts Authentication=ActiveDirectoryDefault, a value the ODBC driver's own Authentication keyword does not offer at all.", + "value_probe": { + "task": "I have a FastAPI service that reads from a database in Azure. It uses pyodbc with the Microsoft ODBC Driver 18 for SQL Server, which is installed in the container image and works fine with a user name and password.\n\nWe are removing the password, so I changed the connection string to this:\n\n```python\ncnxn = pyodbc.connect(\n 'Driver={ODBC Driver 18 for SQL Server};'\n 'Server=tcp:ourserver.database.windows.net,1433;'\n 'Database=orders;'\n 'Authentication=ActiveDirectoryDefault;'\n 'Encrypt=yes;'\n)\n```\n\nIt fails instantly. Not after a timeout, instantly, and the message is about the connection string rather than about a login. I proved it is not the network by pointing the same call at a made up host on a closed port, and I get the identical failure just as fast. The azure-identity package is installed and `az login` works in that shell.\n\nWhat is going on, and what is the right way to make this service connect without a password from Python?", + "gate": "The answer explains that the driver named in this connection string does not accept that authentication value at all and rejects the string before opening a socket, and gives a passwordless route for Python that actually works, whether by supplying an access token to the driver or by using a driver that does accept the value.", + "states": [ + "The failure happens during connection string validation, before any network connection is attempted, which is why an unreachable host produces the same immediate result.", + "The authentication value used here is not one this driver's Authentication keyword accepts, so no amount of identity or network configuration changes the outcome.", + "A passwordless connection on this driver path works by obtaining a token separately and handing it to the driver, rather than by naming an authentication mode in the string.", + "There is a first party Python driver that does accept this authentication value directly and requires no separately installed operating system driver." + ], + "refutes": [ + "The authentication value used here is a valid keyword value for this ODBC driver, so the failure must lie in the identity, the network, the firewall, or the permissions.", + "Installing, upgrading, or reinstalling the operating system ODBC driver, or installing an extra Python package, makes this connection string valid.", + "The immediate failure indicates that no credential was found in the environment, and running an interactive sign in or setting identity environment variables would resolve it.", + "Adding a longer timeout, a different port, or a firewall rule is relevant to a failure that returns this quickly." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-to-azure-sql", + "diagnose-connection-errors", + "sqlalchemy-azure-sql", + "django-azure-sql", + "entra-id-auth" + ], + "validation": { + "target": "both", + "setup": [ + "Python 3.10 or later for mssql-python, or Python 3.9 or later for pyodbc", + "a reachable Azure SQL database, or the local container" + ], + "assert": [ + "new code uses mssql-python and installs no operating system ODBC package", + "a pyodbc path installs msodbcsql18 and names Driver 18 in the connection string", + "mssql_python.pooling() is called before the first connection or not at all", + "Entra ID on the pyodbc path passes a token through SQL_COPT_SS_ACCESS_TOKEN, not Authentication=ActiveDirectoryDefault", + "no credential appears in the connection string on either path" + ], + "probes": [ + { + "where": "host", + "cmd": "python3 -c \"import pyodbc, sys\nsys.exit(0 if 'ODBC Driver 18 for SQL Server' in pyodbc.drivers() else 1)\"", + "expect": { + "succeeds": true + }, + "why": "The skill's first check, made executable. pyodbc installs from a wheel with no compiler step, so pip succeeding proves nothing about the operating system driver the connection string names; this asserts the driver manager has registered that exact name. Measured 2026-09-03 on pyodbc 5.3.0 with unixODBC 2.3.14: exit 0, drivers were ODBC Driver 18 and ODBC Driver 17 for SQL Server. Host, because the claim is about the machine the Python application runs on, not about the engine image. Opens no database connection." + }, + { + "where": "host", + "cmd": "python3 -c \"import pyodbc\ntry:\n pyodbc.connect('Driver={ODBC Driver 18 for SQL Server};Server=tcp:127.0.0.1,1;Authentication=ActiveDirectoryDefault;Encrypt=yes;', timeout=2)\nexcept pyodbc.Error as e:\n print(e.args[0])\" | grep -q '^08001$'", + "expect": { + "succeeds": true + }, + "why": "The central correction of this skill, asserted rather than described. ActiveDirectoryDefault is a mssql-python and azure-identity value; the ODBC driver's Authentication keyword does not have it, and an agent that writes it into a pyodbc string has invented a value. Measured 2026-09-03: SQLSTATE 08001, 'Invalid value specified for connection string attribute Authentication', raised by the driver before it opens a socket, which is why 127.0.0.1 on a closed port is a sufficient target and no engine is needed. SQLSTATE rather than a Msg number because this never reaches the engine." + }, + { + "where": "host", + "cmd": "python3 -c \"import pyodbc\ntry:\n pyodbc.connect('Driver={ODBC Driver 18 for SQL Server};Server=tcp:127.0.0.1,1;Authentication=ActiveDirectoryMsi;Encrypt=yes;', timeout=2)\nexcept pyodbc.Error as e:\n print(e.args[0])\" | grep -q '^HYT00$'", + "expect": { + "succeeds": true + }, + "why": "The control for the probe above, and the reason its result means what the skill says. A driver that rejected every Authentication value would produce 08001 for both. ActiveDirectoryMsi is accepted, gets past string validation and fails later on the network instead. Measured 2026-09-03: SQLSTATE HYT00, login timeout expired, after the two second timeout. If this ever returns 08001 the driver has dropped managed identity support and the skill's production recommendation is wrong." + }, + { + "where": "host", + "cmd": "python3 -c \"import pyodbc, struct\npacked = 'abcd'.encode('utf-16-le')\ntok = struct.pack('- + Connects a TypeScript or JavaScript application to Azure SQL Database with the mssql package + over tedious: which packages to install, where the connection pool has to live, how to type + query results, and how to authenticate with Microsoft Entra ID without a password. Use when a + user says "connect my Node app to Azure SQL", "which npm package for SQL Server", "mssql pool + size", "passwordless connection from Node", "azure-active-directory-default", "@types/mssql", or + reports that a Node API gets slower under load, opens a connection per request, or runs out of + connections. Also use when adding a database layer to a Node backend, an Azure Function or an + Express route. Covers driver installation, the config object, pool lifetime and Entra ID for + this stack only. Encryption doctrine, retry and transient-fault handling belong to + connect-to-azure-sql; Python and .NET have their own skills. +--- + +# Connect from TypeScript and Node.js + +The package is `mssql`, the pool belongs to the module and not the request handler, and the types +come from a second package. + +Verified on 2026-09-03 against `mssql` 12.7.0, `tedious` 20.0.0 and `@types/mssql` 12.3.0 on +Node 24.15.0, resolved in a running process. + +## The stack + +| Package | What it is | +|---|---| +| `tedious` | The TDS protocol implementation. Microsoft names it the Node.js driver for SQL Server and contributes to it, but states it is community-supported with no Microsoft support | +| `mssql` | The wrapper almost every application uses: pooling, parameterised requests and transactions, over `tedious` | +| `@types/mssql` | The TypeScript definitions. `mssql` publishes **no** `types` field of its own | + +```bash +npm install mssql +npm install --save-dev @types/mssql +``` + +`mssql` 12.7.0 accepts `tedious` `^19.2.2 || ^20.0.0` and declares `engines.node >= 18.19.0`; +`tedious` 20.0.0 declares `engines.node >= 22`. Node 22 or later resolves to `tedious` 20; +Node 18 quietly stays on 19. Pin the Node version in `package.json` and CI. + +## The config object + +```ts +import sql from 'mssql'; + +export const config: sql.config = { + server: process.env.AZURE_SQL_SERVER!, // .database.windows.net + port: 1433, // a number, never a string + database: process.env.AZURE_SQL_DATABASE!, + options: { encrypt: true, trustServerCertificate: false }, + pool: { max: 10, min: 0, idleTimeoutMillis: 30000 }, +}; +``` + +- **`port` must be a number.** The config is not coerced, and a string port fails in a way that + reads like a network fault. +- **At these versions both packages default `encrypt` to `true` and `trustServerCertificate` to + `false`.** `mssql` seeds the pair before handing `options` to `tedious`, and `tedious` 20 resolves + the same two on its own, so a pool built with an empty `options` still resolves to that pair. + Earlier majors disagreed, so pin the versions rather than trust the default. At these versions a + wrong value is something a person typed. When either may legitimately change is `connect-to-azure-sql`. +- **The `pool` values above are the defaults.** Ten is a floor for a toy and a ceiling for a real + API, so set `max` deliberately. Choosing the number, and why the binding limit is workers rather + than sessions, is `connect-to-azure-sql`. + +## The pool is a module-level object + +This is the failure this skill exists for. + +`sql.connect(config)` does not open a connection. It constructs a **process-wide `ConnectionPool` +the first time it is called** and returns that same pool on every later call. + +```ts +// db.ts: one pool, created once, awaited by everything else +import sql from 'mssql'; +import { config } from './config.js'; + +export const poolPromise = new sql.ConnectionPool(config).connect(); +``` + +Every handler does `const pool = await poolPromise;` and nothing more. + +What goes wrong when the pool moves into the handler: + +- **A new `ConnectionPool` per invocation** pays a TCP connect, a TLS handshake and a login every + request, and each pool holds its own sockets, so the process ends far over its intended count. +- **`sql.close()` at the end of a handler** destroys the global pool under requests still using it, + and against a `new sql.ConnectionPool` it closes only the pool `sql.connect()` built, returns + `null` and leaves the sockets open. Use `pool.close()`, on a shutdown path and nowhere else. +- **In a serverless host**, module scope survives warm invocations and the handler body does not, + which is what makes reuse possible. + +## Microsoft Entra ID without a secret + +`mssql` passes `authentication` straight through to `tedious`. The types `tedious` 20 accepts, from +the shipped `connection.d.ts`: + +| `authentication.type` | Use it for | +|---|---| +| `azure-active-directory-default` | Local development and most Azure hosts. Walks the `@azure/identity` chain | +| `azure-active-directory-msi-app-service`, `azure-active-directory-msi-vm` | A managed identity | +| `azure-active-directory-service-principal-secret` | An app registration and client secret | +| `azure-active-directory-access-token` | A token the app acquired itself | +| `token-credential` | An `@azure/identity` credential supplied directly | +| `azure-active-directory-password` | Nothing. Microsoft has deprecated this flow | + +```ts +export const entraConfig: sql.config = { + server: process.env.AZURE_SQL_SERVER!, + database: process.env.AZURE_SQL_DATABASE!, + options: { encrypt: true, trustServerCertificate: false }, + authentication: { + type: 'azure-active-directory-default', + options: { clientId: process.env.AZURE_CLIENT_ID }, // user-assigned identity only + }, +}; +``` + +`@azure/identity` is a dependency of `tedious`, so there is nothing extra to install. This config +holds no secret and is safe in source control; the SQL authentication variant is not. The identity +still needs a database principal, which is `entra-id-auth`. + +## Typed results + +```ts +interface Person { id: number; first_name: string; } + +const result = await pool.request() + .input('id', sql.Int, id) + .query('SELECT id, first_name FROM person WHERE id = @id'); + +const rows: Person[] = result.recordset; // recordsets for every set, rowsAffected[0] for counts +``` + +The generic on `query()` is a **compile-time assertion, not runtime validation**. Nothing checks +the columns against `Person`; a renamed column yields `undefined` at run time while the types +compile. If the shape matters, validate the rows or generate the types from the schema. + +Use `request.input(name, type, value)` for every value; concatenating into the SQL text is how +injection gets in. + +## Check it worked + +Ask through `mssql` itself: what a command line client proves about encryption is true of its own +session, not of the pool your process opened. Save this beside `db.ts` as `check-connection.mjs`. + +```js +import { poolPromise } from './db.js'; + +const pool = await poolPromise; +const { recordset: [r] } = await pool.request().query(` + SELECT c.encrypt_option, DB_NAME() AS db, + SUSER_NAME() AS login, USER_NAME() AS db_user + FROM sys.dm_exec_connections AS c WHERE c.session_id = @@SPID`); +console.log(r, 'pool:', pool.size, pool.available, pool.borrowed); +await pool.close(); +``` + +```bash +node check-connection.mjs +``` + +| Read | Expect | A wrong value means | +|---|---|---| +| `encrypt_option` | `TRUE` | the session is in clear text; someone overrode `options.encrypt` | +| `db` | your database, never `master` | `database` is missing from the config | +| `login`, `db_user` | the principal you granted rights to | the credential chain picked another one | +| `pool.size` | at most `pool.max`, and flat as requests arrive | a pool per handler, the failure above | + +Two things this check trips over, measured 2026-09-03: + +- **`pool.size` throws before the pool is awaited**, `TypeError: Cannot read properties of + undefined (reading 'numFree')`, while `@types/mssql` declares it `readonly size: number`. Read it + after the await or the check dies on its own instrumentation. +- A wrong host or port stops the script first, as `ConnectionError: Failed to connect to + :`. + +`sys.dm_exec_connections` needs `VIEW DATABASE STATE`, and on Basic, S0, S1 and elastic pools the +server administrator, the Microsoft Entra administrator or `##MS_ServerStateReader##`. Grant it for +the check, then revoke it. `CONNECTIONPROPERTY` is no substitute: its documented property list has no +`encrypt_option`. + +## Do not + +- Do not call `sql.connect()` or construct a pool inside a request handler, route or serverless + function body. +- Do not call `pool.close()` outside a deliberate shutdown path. +- Do not set `trustServerCertificate: true` to clear a certificate error. Read + `connect-to-azure-sql`. +- Do not use `azure-active-directory-password`. Microsoft has deprecated the flow it uses. +- Do not write retry loops here. Transient faults are one policy for every language, in + `connect-to-azure-sql`. + +## References + +- [Node.js driver for SQL Server](https://learn.microsoft.com/sql/connect/node-js/node-js-driver-for-sql-server): + what `tedious` is and what support it carries. Read it when asked which driver is official. +- [Connect and query with Node.js and mssql](https://learn.microsoft.com/azure/azure-sql/database/azure-sql-javascript-mssql-quickstart): + the first-party quickstart, passwordless configuration included. Read it for a working end-to-end + sample. +- [node-mssql documentation](https://tediousjs.github.io/node-mssql/): the config, pool and request + API in full. Read it when a config key is in question. +- [sys.dm_exec_connections](https://learn.microsoft.com/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-connections-transact-sql): + the columns and the permission the check needs. Read it when the check returns no row. +- `connect-to-azure-sql`: encryption doctrine, retry and transient faults, pool sizing, and the + first-connect error on a paused database. +- `connect-from-edge-runtimes`: for a runtime without TCP sockets, where `mssql` does not run. diff --git a/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node/skill.spec.jsonc new file mode 100644 index 0000000..686a70c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-from-typescript-and-node/skill.spec.jsonc @@ -0,0 +1,153 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "connect-from-typescript-and-node", + "domain": "drivers-and-connectivity", + "value": [ + "corrects-a-wrong-assumption", + "changes-too-fast-to-memorize" + ], + "correction": "Agents give every request or serverless invocation its own mssql connection pool, and close it when the handler returns, because they read sql.connect() as opening one connection; it actually creates a process-wide pool whose default ceiling is ten connections, so the per-invocation version pays a full TLS and login handshake on every call and the paired sql.close() tears the pool out from under every other in-flight request.", + "value_probe": { + "task": "Our Express API on Node.js 20 talks to a database in Azure through the mssql package, pinned at a current version. Every route calls this helper:\n\n```ts\nexport async function query(text: string): Promise {\n const pool = await sql.connect(config);\n const result = await pool.request().query(text);\n await sql.close();\n return result.recordset;\n}\n```\n\nUnder light load it is fine. Under real traffic two things happen. Response times climb well past what the queries themselves take, and we get intermittent failures on requests that were already in flight, complaining that the connection is closed. The failures are not tied to any one route, and the same queries are fast when run from a command line client.\n\nOne engineer wants to raise the pool maximum, another wants to raise the connection timeout. Neither has helped in staging. Show me the corrected helper and explain what was actually wrong.", + "gate": "The answer removes the per request creation and teardown, leaving one shared pool that is created once for the life of the process and reused by every request, so no request can destroy the connections another request is using.", + "states": [ + "The connect call does not open a single dedicated connection; it creates or returns a process wide pool shared by every caller.", + "The close call in this helper destroys that shared pool, which is why requests already in flight fail with a closed connection.", + "The latency growth comes from paying a full encryption and login handshake on requests that should have reused an already open connection.", + "The corrected shape creates the pool once at module level and exports it, with no close call in the request path." + ], + "refutes": [ + "The connect call opens one connection belonging to this request, so closing it when the handler finishes affects only that request and is correct practice.", + "The problem is that the pool maximum or the connection timeout is set too low, and raising either of them is the fix.", + "The closed connection errors come from the server dropping idle connections, so a keep alive setting or a retry wrapper around the query is what is needed.", + "Leaving connections open across requests is a leak, so a connection must be released or closed at the end of every handler." + ] + }, + "posture": [ + "read", + "write", + "execute" + ], + "maturity": "ga", + "applies_to": [ + "azure-sql-db", + "azure-sql-db-container" + ], + "related": [ + "connect-to-azure-sql", + "diagnose-connection-errors", + "connect-from-edge-runtimes", + "typeorm-and-sequelize-azure-sql", + "prisma-azure-sql" + ], + "validation": { + "target": "both", + "setup": [ + "a Node.js 22 or later project with mssql installed", + "a reachable Azure SQL database, or the local container" + ], + "assert": [ + "exactly one ConnectionPool is constructed per process, at module scope", + "the resolved tedious options report encrypt true and trustServerCertificate false", + "pool.max is set deliberately rather than left at the default of 10", + "no request handler calls sql.close()", + "@types/mssql is a devDependency when the project is TypeScript" + ], + "probes": [ + { + "where": "host", + "cmd": "npm view mssql@12.7.0 dependencies.tedious | grep -qxF '^19.2.2 || ^20.0.0'", + "expect": { + "succeeds": true + }, + "why": "The body states the exact tedious range mssql 12.7.0 accepts, and the whole Node version split rests on it. Read from the published manifest rather than recalled. Measured 2026-09-03 on this host: the command prints '^19.2.2 || ^20.0.0' and exits 0. Host, because this is a registry read by a developer's own npm and opens no database connection." + }, + { + "where": "host", + "cmd": "npm view mssql@12.7.0 engines.node | grep -qx '>=18.19.0'", + "expect": { + "succeeds": true + }, + "why": "One half of the version split the body tells a reader to pin against. If mssql raised its floor to 22 the advice would be stale rather than wrong, and this is what would say so. Measured 2026-09-03: prints '>=18.19.0'. Host, a registry read with no engine involved." + }, + { + "where": "host", + "cmd": "npm view tedious@20.0.0 engines.node | grep -qx '>=22'", + "expect": { + "succeeds": true + }, + "why": "The other half. tedious 20 demanding Node 22 is why an install on Node 18 silently stays on tedious 19, which is the only reason the body tells anyone to pin the Node version in package.json and CI. Measured 2026-09-03: prints '>=22'. Host, a registry read." + }, + { + "where": "host", + "cmd": "test \"$(npm view mssql@12.7.0 name)\" = 'mssql' && test -z \"$(npm view mssql@12.7.0 types)\"", + "expect": { + "succeeds": true + }, + "why": "The body claims mssql publishes no types field of its own, which is the entire reason @types/mssql is a separate devDependency. Asserting an absence, so the first test proves npm answered at all before the second reads an empty value; without it a network failure would look like a pass. Measured 2026-09-03: name is mssql, types is empty, and the full manifest carries neither types nor typings." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && npm install --prefix \"$d\" --no-audit --no-fund --silent mssql@12.7.0 >/dev/null 2>&1 && node -e 'const sql=require(process.argv[1]+\"/node_modules/mssql\");const o=new sql.ConnectionPool({server:\"h\",database:\"d\"})._config().options;process.exit(o.encrypt===true&&o.trustServerCertificate===false?0:1)' \"$d\"; e=$?; rm -rf \"$d\"; exit $e", + "expect": { + "succeeds": true + }, + "why": "The encryption default, resolved in a running process rather than read off a page. A pool built with no options at all must come out encrypt true and trustServerCertificate false, which is what lets the body say a wrong value at these versions is something a person typed. Earlier majors of both packages did not agree, so the version is pinned into the command. Measured 2026-09-03 on mssql 12.7.0 with tedious 20.0.0: resolved true and false, exit 0. Host, because this is a client library on a developer's machine and no database is contacted." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && npm install --prefix \"$d\" --no-audit --no-fund --silent mssql@12.7.0 >/dev/null 2>&1 && node -e 'const sql=require(process.argv[1]+\"/node_modules/mssql\");try{new sql.ConnectionPool({server:\"h\",database:\"d\"}).size;process.exit(1)}catch(e){process.exit(e instanceof TypeError && /numFree/.test(e.message)?0:1)}' \"$d\"; e=$?; rm -rf \"$d\"; exit $e", + "expect": { + "succeeds": true + }, + "why": "The trap in the body's own check script. @types/mssql declares size as readonly size: number, so TypeScript permits reading it, and reading it before the pool is awaited throws TypeError: Cannot read properties of undefined (reading 'numFree'). The probe passes when it throws that exact TypeError, so a future release that makes size safe before connect turns this red and the body's warning can be dropped. Measured 2026-09-03 on mssql 12.7.0." + }, + { + "sql": "SELECT COUNT(*) AS rows_for_my_own_session FROM sys.dm_exec_connections WHERE session_id = @@SPID;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The check script's query has to return a row or the whole Check it worked section is theatre. This asserts existence rather than encrypt_option on purpose: encrypt_option describes whichever client opened the session, so a value read here would describe the lane's own sqlcmd connection and not the pool the reader's process opened, which is the exact confusion the section is written to prevent." + }, + { + "sql": "SELECT CASE WHEN CONNECTIONPROPERTY('encrypt_option') IS NULL THEN 1 ELSE 0 END AS connectionproperty_has_no_encrypt_option;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The body tells a reader that CONNECTIONPROPERTY is no substitute for sys.dm_exec_connections because its documented property list carries no encrypt_option, and Microsoft Learn states any other string returns NULL. That is sourced from Learn and NOT yet measured against this engine, which is why it is a probe: if it comes back non-null the body's advice needs rewriting rather than defending." + } + ] + }, + "triggering": { + "explicit": [ + "Use the connect-from-typescript-and-node skill", + "Connect my Node.js app to Azure SQL Database", + "Which npm package do I use to talk to Azure SQL from TypeScript" + ], + "implicit": [ + "my Express API opens a new database connection on every request and gets slower under load", + "connect to Azure SQL from an Azure Function in TypeScript without a password", + "how do I type the rows coming back from a SQL query in TypeScript", + "set pool size for the mssql package" + ], + "routing": [ + "add a database layer to my Node backend", + "passwordless SQL connection for a Node app on App Service", + "add a database layer to my TypeScript app on Azure SQL" + ], + "negative": [ + "connect to Azure SQL from Python with pyodbc", + "set up Microsoft.Data.SqlClient managed identity in my .NET API", + "my first connection to a paused serverless database fails with error 40613", + "run a query from the command line against my database" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql deleted file mode 120000 index f271ea1..0000000 --- a/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/connect-to-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/SKILL.md new file mode 100644 index 0000000..0ceed53 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/SKILL.md @@ -0,0 +1,337 @@ +--- +name: connect-to-azure-sql +description: >- + Connects an application to Azure SQL Database: picks the Microsoft driver for the language, + sets encryption and certificate validation, sizes the pool against the worker limit not the + session limit, and makes retry part of the first version of the code. Use when a user asks "how + do I connect to Azure SQL", "which driver should I use", "what goes in the connection string", + "add retry logic", "the connection keeps dropping", "should I set TrustServerCertificate", or + "my app times out connecting to Azure". Also use when a first connection to a free or + serverless database fails with error 40613, documented resume behaviour, not an outage. + Azure SQL Database only, not Azure SQL Managed Instance and not self-managed SQL Server. Driver + installation, connection strings and pooling per language belong to connect-from-dotnet, + connect-from-python and connect-from-typescript-and-node. +--- + +# Connect to Azure SQL Database + +Opening one connection is easy. Two things go wrong: **the first connection is expected to fail and +the code has no retry**, and **a certificate error gets fixed by turning verification off**. + +Flags and defaults below were verified 2026-09-03 against `sqlcmd` 1.10.0 and User Secrets Manager +8.0.27 on this machine; cloud behaviour is from Microsoft Learn. + +## Connect once from the command line before writing any code + +Settle the server, database, credential and certificate before an application is in the picture, so +every later failure is a failure of the code. Cloud, SQL authentication, password out of the +command line: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -P "$SQL_PASSWORD" \ + -N mandatory -l 30 -Q "SELECT DB_NAME() AS db, encrypt_option FROM sys.dm_exec_connections WHERE session_id = @@SPID" +``` + +Cloud, Microsoft Entra ID, with no password anywhere: + +```bash +az login +sqlcmd -S .database.windows.net,1433 -d -G -N mandatory -l 30 \ + -Q "SELECT SUSER_NAME() AS login" +``` + +Local Azure SQL Database container. **`-C` is the entire difference**, there because the container +presents a self-signed certificate. + +```bash +sqlcmd -S 127.0.0.1,1433 -d -U sa -P "$MSSQL_SA_PASSWORD" \ + -N mandatory -C -l 30 -Q "SELECT DB_NAME() AS db" +``` + +Each switch is a connection string keyword under another name, so a working command line converts +straight into a working string: + +| Switch | Connection string keyword | Note | +|---|---|---| +| `-N mandatory` | `Encrypt=Mandatory` | `-N strict` is TDS 8.0, where trust cannot be bypassed | +| `-C` | `TrustServerCertificate=true` | Container only | +| `-F ` | `HostNameInCertificate` | The fix for a name mismatch | +| `-J ` | certificate pinning, PEM, DER or CER | Validates one certificate, not none | +| `-l 30` | `Connect Timeout` | Covers the retries inside it | +| `-G` | `Authentication=Active Directory Default` | No `-U`; add `-U` for interactive | + +`-N` is validated before any connection is attempted, so a typo is caught for free. Measured +2026-09-03 on 1.10.0: any other value exits 1 with `Argument value has to be one of [m[andatory] +yes 1 t[rue] disable o[ptional] no 0 f[alse] s[trict]]`. + +## Container and cloud: the same string with one exception + +| | Local container | Azure SQL Database | +|---|---|---| +| Server | `127.0.0.1,` | `.database.windows.net,1433` | +| Encryption | on | on | +| Certificate | self-signed, so `-C` / `TrustServerCertificate=true`, or pin it with `-J` | validated, never `-C` | +| Authentication | SQL login. Microsoft Entra ID only if the container was started with an application registration certificate, a client id and a tenant id: below | SQL login or Entra ID | +| Retry for `40613` | no equivalent; nothing throttles or fails over locally | mandatory, below | +| Pool ceiling | no service tier, so no worker number to size against | size against workers | + +A clean local run is therefore no evidence the retry story works. + +### Microsoft Entra ID on the container is configured, not default + +The image reads the certificate path, the application (client) id and the tenant id from three +`MSSQL_AAD_` environment variables set when the container is created. Learn documents that trio for +SQL Server on Linux containers and does not document this image, so here it is measured, not cited: + +```bash +docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' | grep -c '^MSSQL_AAD_' +``` + +`3` or more means Entra ID was configured. Measured 2026-09-03 on a container created without it: +`0`, exit 1. **That state refuses the login, not the mode**, so `-G` or +`Authentication=Active Directory Default` returns `18456`, `Login failed for user ''` (measured +2026-08-28 in `connect-from-dotnet`), naming neither Entra ID nor the missing configuration, and no +connection string edit fixes it. Open `entra-id-auth` once the three are set and the identity needs +a database user, or when an `18456` names a token-identified principal rather than an empty one. + +## What this skill does not own + +Driver installation, per-language syntax and pool configuration are `connect-from-dotnet`, +`connect-from-python`, `connect-from-typescript-and-node`. An error number needing a cause is +`diagnose-connection-errors`, except 40613, which stays here. Resource pressure is +`diagnose-resource-pressure`, and creating the server and its firewall rule is +`provision-azure-sql-db`. + +## Retry is part of the first version, not the hardening pass + +Azure SQL Database reconfigures under you. Microsoft's wording: most reconfiguration events +complete in under 60 seconds, and applications **should be built to expect these transient errors** +rather than surface them to users. + +On General Purpose serverless, which is what the free offer runs on, Microsoft documents this: + +> If a serverless database is paused, the first connection attempt resumes the database and returns +> an error stating that the database is unavailable with error code 40613. Once the database +> resumes, retry the connection. Databases generally resume in less than one minute. + +The default auto-pause delay is 60 minutes, so a developer back from lunch gets 40613 on the first +run and reads it as a wrong password or a dead server. **Code that leaves retry for later fails the +first time it is run.** + +### The retry policy + +- Wait **5 seconds** before the first retry. Shorter risks overwhelming the service. +- Grow the delay **exponentially**, to a maximum of **60 seconds**, and cap the attempts so a broken + configuration eventually reports itself. +- **A transient error during a query is not retried on the same connection.** Open a fresh + connection, then retry the command, and confirm the transaction committed or rolled back first. + +### Transient, so retry + +| Number | What it is | +|---|---| +| `40613` | Database not available: resume of a paused database, reconfiguration, or the dedicated administrator connection already in use | +| `40197` | The service hit an error processing the request. Carries an embedded code worth logging | +| `40501` | The service is busy. Engine throttling | +| `40540`, `40143` | Reconfiguration and failover paths | +| `10928`, `10929` | A resource limit was reached. Resource ID 1 is workers, Resource ID 2 is sessions | +| `49918`, `49919`, `49920` | Not enough resources, or too many control-plane operations in flight | +| `233`, `64`, `20`, `10053`, `10054`, `10060` | Transport-level failures during or just after connect | + +`4060` is on the transient list too, but usually means the database name is wrong or the login has +no user there. Retry it a small fixed number of times, then report it. + +### Not transient, so do not retry + +| Number | Cause | Fix | +|---|---|---| +| `18456` | Login failed | Credential or user problem, not a service problem | +| `40615` | Client IP not allowed | A firewall rule is missing. See `provision-azure-sql-db` | +| `47073` | Public network access is disabled on the server | Connect through the private endpoint | +| `47072` | Login failed with invalid TLS version | The client negotiated below the server's minimum | + +One case **looks** transient and is not: a free-offer database whose monthly allowance is exhausted +is inaccessible **until the start of the next calendar month**, so retry never succeeds. + +### Do not stack retry on retry without doing the arithmetic + +Driver connection resiliency multiplies with application-level retry. The documented .NET example: +application logic retrying 4 times over a driver configured for 3 connection retries is **12** +attempts. + +The .NET defaults an unconfigured string already has: `ConnectRetryCount` **1**, +`ConnectRetryInterval` **10** seconds, `Connection Timeout` **15** seconds, `Command Timeout` **30** +seconds. Learn states two inequalities that hold whatever the language: + +- New connection: `Connection Timeout >= ConnectRetryCount * ConnectRetryInterval`, or the outer + timeout cancels the last retry. At 3 retries of 10 seconds, a timeout of 29 never reaches the + third. +- Reconnect during command execution: `Command Timeout > (ConnectRetryCount - 1) * + ConnectRetryInterval`, plus time for the command itself. + +Pick one layer to own retry and turn the other off deliberately. **Two Learn pages disagree about +what `ConnectRetryCount` covers**: the transient-errors article puts the initial `Open()` inside it, +the SqlClient overview calls it idle resiliency only. Assume the narrower reading and write your own +retry around the first connect. + +## Encryption is not a knob to turn off + +Microsoft's recommendation for every connection to Azure SQL Database, in every driver: +`Encrypt = On`, `TrustServerCertificate = Off`, plus `HostNameInCertificate = ` when +the client connects through another name such as a DNS alias. Together they make the driver verify +the server's identity. `TrustServerCertificate=true` keeps the traffic encrypted but **stops the +client checking who it is talking to**, the part that matters on a public endpoint. Encryption and identity are +separate settings, and login credentials are encrypted whatever `Encrypt` says. + +**The rule for an agent: a certificate error is a name problem, not a trust problem.** Fix the +hostname or set the hostname-in-certificate option. Never disable verification, never set +`Encrypt=false`; the one exception is the container's self-signed certificate above. + +Two facts that change under the model's feet: + +- **The defaults flipped.** `Microsoft.Data.SqlClient` 4.0 changed `Encrypt` from `false` to `true` + and ODBC Driver 18 made the same change, both declared breaking. Code that worked before an + upgrade and fails after it is usually this, surfacing as + `SSL Provider: The certificate chain was issued by an authority that is not trusted`. +- **TLS 1.0 and 1.1 are retired.** The lowest minimum a server accepts is TLS 1.2, enforcing a + specific minimum cannot be reverted, and enforcing 1.3 breaks clients whose driver or operating + system does not support it. + +## Choosing a driver + +Use the driver Microsoft publishes for the language. Anything else is a support problem later, and +non-Microsoft drivers may not use TLS by default. + +| Language | Driver | +|---|---| +| C# and .NET | `Microsoft.Data.SqlClient` | +| Python | `mssql-python` | +| Node.js and TypeScript | the Node.js driver for SQL Server | +| Java | the Microsoft JDBC driver | +| Go | `go-mssqldb` | +| C and C++ | ODBC Driver 18 for SQL Server | +| PHP | the Microsoft PHP drivers | +| Ruby | the Ruby driver for SQL Server | + +## What the connection string carries + +**Must have:** the fully qualified server name and port `1433`, the **user database** and never +`master`, encryption on with certificate trust off, and a connect timeout with room for the retries +inside it. + +**Must not have:** + +- a password literal. The secret comes from a secret store or the environment at run time: User + Secrets locally, a vault in production, never a checked-in file. + + ```bash + dotnet user-secrets init --project ./src/Api + dotnet user-secrets set "ConnectionStrings:Default" \ + "Server=tcp:.database.windows.net,1433;Initial Catalog=;Authentication=Active Directory Default;Encrypt=Mandatory;Connect Timeout=60" \ + --project ./src/Api + ``` + +- `TrustServerCertificate=true` outside the local container, or `Encrypt=false` anywhere +- `Authentication=Active Directory Password`. That mode is **deprecated** in the Microsoft SQL + drivers, incompatible with mandatory Microsoft Entra multifactor authentication, and obsolete in + `Microsoft.Data.SqlClient` 7.0. Use managed identity for Azure-hosted workloads, a service + principal off Azure, and interactive when a human is present + +### Gotcha: the generated connection string is not finished + +```bash +az sql db show-connection-string -c ado.net -s -n +``` + +On 2.89.1 the output needs correcting by hand. The ODBC template emits +`Driver={ODBC Driver 13 for SQL Server}`, five major versions out of date. `--auth-type ADPassword` +emits the deprecated `Authentication="Active Directory Password"`. `--client` accepts only +`ado.net`, `jdbc`, `odbc`, `php`, `php_pdo` and `sqlcmd`, so asking it for a Node.js, Python or Go +template and paraphrasing the answer invents a string. And no template carries retry settings. + +## Size the pool against workers, not sessions + +The number an agent reaches for is the session limit, **30,000 on every vCore size**, never the +binding constraint. What throttles a workload is **concurrent workers**: + +| Compute, standard-series | Concurrent workers | +|---|---| +| General Purpose provisioned | **100 per vCore** (200 at 2 vCores, 1000 at 10) | +| General Purpose serverless | **75 per max vCore** (75 at 1, 300 at 4) | + +And workers are not requests. Default `MAXDOP` for a new database is **8**, the limit is **per +task**, and each task takes a worker, so one parallel request consumes several and the ceiling +arrives sooner than a count of connections suggests. + +Set an explicit maximum pool size: unlimited turns a traffic spike into `10928`. Sum the +pools across **every** instance, background workers and scheduled jobs included, and keep the total +under the worker limit with headroom. In serverless and short-lived compute create **one pool per +process** and reuse it; one per request or per invocation exhausts outbound ports and causes +connection storms. A database already failing with `10928` is `diagnose-resource-pressure`. + +## The network path, when it works from one place but not another + +The connection policy decides the route and its default differs by where the client is. **Outside +Azure** it is `Proxy`: everything goes through the gateway and the client needs outbound `1433` +only. **Inside Azure** it is `Redirect`: after the gateway handshake the session moves to the node +hosting the database, so the client needs outbound `1433` **plus 11000 to 11999** to the region's +Azure SQL Database addresses. That asymmetry is why an application works from a laptop and times out +from a virtual machine behind a restrictive network security group. `Redirect` is recommended; the cost is the wider +outbound range. + +```bash +az sql server conn-policy show -g -n +az sql server conn-policy update -g -n --connection-type Redirect +``` + +`--connection-type` accepts `Default`, `Proxy` and `Redirect`. + +## Check it worked + +Three checks, run against the target the application will use, with `-C` added only for the +container. + +**1. The connection is encrypted and landed in the user database.** The first command above asks +the engine itself, so it settles encryption without trusting the client. Expected: `db` is the user +database and not `master`, `encrypt_option` is `TRUE`, and the command succeeding with no `-C` is +itself the proof that the certificate validated. + +**2. A string left on `master` cannot be corrected at run time**, which is why the database belongs +in the connection string: + +```sql +USE master; +``` + +Expected: `Msg 40508, USE statement is not supported to switch between databases.` + +**3. No secret reached the repository.** + +```bash +git grep -nEi "(password|pwd)=[^;\"']" -- . ":!*.md" +``` + +Expected: no output, exit 1. Any hit is a secret in source, whatever else works. + +A fourth check on serverless or free: leave the database idle past the auto-pause delay, run the +application cold, and confirm the code survives `40613`. + +## Read the source when + +Fetch the Microsoft Learn article rather than recalling it when an error number needs a cause (then +`diagnose-connection-errors`), when the retry delays matter to a design (the transient errors +article), when a resource limit is close (the vCore resource limits reference, the only current +source for worker counts), or when a TLS minimum is being set (the connectivity settings article). + +## Do not + +- Do not present retry as an optional later step. `40613` on a first connect is the documented + resume path, not an outage. +- Do not set `TrustServerCertificate=true` or `Encrypt=false` to clear a certificate error against + the cloud, not even temporarily. That change survives to production. +- Do not carry the container's `-C` into the cloud string. It is the one line that must not travel. +- Do not use `Authentication=Active Directory Password`, or copy it out of generated output, and do + not paste a generated ODBC connection string without correcting the driver name. +- Do not size a pool against the 30,000 session limit, or create one per request or per invocation. +- Do not apply any of this to Azure SQL Managed Instance or SQL database in Fabric. Different + connectivity model, different limits. diff --git a/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..5eff65a --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/connect-to-azure-sql/skill.spec.jsonc @@ -0,0 +1,144 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "connect-to-azure-sql", + "domain": "drivers-and-connectivity", + "value": ["corrects-a-wrong-assumption", "changes-too-fast-to-memorize"], + "correction": "Agents write Azure SQL Database connection code as a single attempt and treat retry as a later hardening step, when retry is mandatory rather than optional on this service: the first connection to an auto-paused serverless or free database is documented to fail with error 40613 while the resume happens, so the generated code fails on its very first run and the failure reads as an outage or a bad credential. The same reflex then downgrades security, because an agent that meets a certificate error reaches for TrustServerCertificate=true or Encrypt=false instead of fixing the name it is connecting to.", + "value_probe": { + "task": "We are starting a new order service. It runs against a local Azure SQL Database container on developer machines and against a database in Azure for the deployed environments. Today the team stores one database user name and password in configuration and uses it in both places, and the data access code opens a connection and runs the query with no wrapper around it.\n\nTwo things keep happening. The first call of the morning against the shared development database fails, and the second call a moment later succeeds, so our on-call rota has an entry telling people to \"just try again\" and one engineer has already filed a support case about it. Separately, a developer hit a certificate error connecting from a new laptop and got past it by adding a setting to the connection string, and that connection string has now been copied into the deployed configuration.\n\nWe are about to write the real data access layer. Tell me what it should look like for both environments, what belongs in the connection string, and how we should be handling credentials. Be specific about what changes and why.", + "gate": "The answer treats reconnection through transient failures as part of the connection code from the first version rather than as later hardening, and keeps encryption on with server certificate validation intact in the deployed environment, allowing a trust override only where the local container's self-signed certificate genuinely requires it.", + "states": [ + "A connection that fails on the first attempt and succeeds on a retry a moment later is expected behaviour for a database that can be idle, not evidence of an outage or a bad credential.", + "The connection code must retry transient failures itself, because no configuration change on the server removes them.", + "A certificate validation error is fixed by correcting what the client connects to or what it trusts, not by telling the client to accept any certificate.", + "The deployed environment should authenticate without a stored database password, using the platform identity of the running service." + ], + "refutes": [ + "A first connection failing while an immediately following one succeeds indicates a service outage, an incorrect password, or a misconfigured server, and should be escalated rather than retried.", + "Retry for transient failures is a hardening step to add once the connection code is working, rather than part of the first version of it.", + "Turning off server certificate validation, or turning encryption off, is an acceptable way to resolve a certificate error in a deployed environment.", + "The same stored user name and password is the correct credential for both the local container and the deployed environment, because no other option works in both." + ] + }, + "posture": ["read", "write"], + // Held at draft below a preview ceiling, on purpose. The value measurement of + // 2026-09-05, ten runs per arm, returned NEITHER: every check reads level, and + // the gate 9 of 10 with the skill against 8 of 10 without. The model already + // answers this skill's own task correctly without it. + // + // READ THIS BEFORE TOUCHING THE CONNECT FAMILY. This skill is the retry and + // encryption doctrine that connect-from-dotnet, connect-from-python and + // connect-from-typescript-and-node defer to rather than restate. Draft keeps it + // out of the catalog, so those three now point at a skill a reader cannot + // install. Demoting it was Carlos's call on 2026-09-20 and the cross-reference + // question is open, not settled. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db"], + "related": [ + "connect-from-dotnet", + "connect-from-python", + "connect-from-typescript-and-node", + "diagnose-connection-errors", + "diagnose-resource-pressure", + "provision-azure-sql-db", + "entra-id-auth", + "sqlcmd-cli" + ], + "validation": { + "target": "both", + "setup": [ + "a General Purpose serverless database with auto-pause enabled, left idle past its auto-pause delay", + "a client with the server firewall already open, so a failure is not a firewall failure" + ], + "assert": [ + "the first connection to the paused database fails with error 40613 and the retry succeeds within about one minute", + "the connection string sets encryption on and certificate trust off, and carries no password literal", + "the connect timeout is at least the retry count multiplied by the retry interval, so the last retry is not cancelled by the outer timeout", + "a login failure (18456) or a firewall rejection (40615) is not retried", + "Microsoft Entra ID against the container is claimed only for a container started with the application registration certificate, client id and tenant id, and 18456 is named as the symptom of the unconfigured case", + "the pool ceiling is sized against the worker limit, not against the 30,000 session limit" + ], + "probes": [ + { + "where": "host", + "cmd": "sqlcmd --version", + "expect": { "succeeds": true }, + "why": "Every command in this skill is a sqlcmd command line, so the tool being installed and printing its build number with no database in reach is the precondition for the five probes below. This one is host rather than container deliberately: the reader of this skill is a developer connecting FROM their machine, and the client half is what the skill is about. Measured 2026-09-03, exit 0, version 1.10.0. If this goes red the others are meaningless." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' | grep -q '^-C,--trust-server-certificate$'", + "expect": { "succeeds": true }, + "why": "The switch the skill tells a reader to use against the local container and never against the cloud, checked to exist by that exact name out of the build's own help rather than from memory. Measured 2026-09-03 on 1.10.0. Host, because this is the client tool a developer installs; the engine image ships its own ODBC sqlcmd and this claim is about the one on the workstation." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' | grep -q '^-N,--encrypt-connection$'", + "expect": { "succeeds": true }, + "why": "The encryption switch that maps to the Encrypt connection string keyword. The skill's whole correction is that encryption stays on while certificate trust is the thing being discussed, so a build that renamed or dropped this switch would make the connection commands in the body wrong. Measured 2026-09-03 on 1.10.0. Host, same reason as above." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' | grep -q '^-F,--host-name-in-certificate$'", + "expect": { "succeeds": true }, + "why": "The skill's answer to a certificate error is to fix the name rather than disable verification, and this is the switch that does it. If it did not exist, the advice would have no implementation and the reader would be pushed back to -C. Measured 2026-09-03 on 1.10.0. Host, same reason as above." + }, + { + "where": "host", + "cmd": "sqlcmd '-?' | grep -q 'ActiveDirectoryDefault'", + "expect": { "succeeds": true }, + "why": "The skill routes a passwordless cloud connection through -G, whose documented behaviour with no -U is ActiveDirectoryDefault, and states that Active Directory Password is deprecated. This checks the mode name is still one the tool offers. Measured 2026-09-03 on 1.10.0, in the --authentication-method list. Host, same reason as above." + }, + { + "where": "host", + "cmd": "sqlcmd -S 127.0.0.1,$PROBE_PORT -U sa -P probe -N banana -l 1 -Q 'SELECT 1' 2>&1 | grep -q 'Argument value has to be one of'", + "expect": { "succeeds": true }, + "why": "An invented encryption value is refused by name before any connection is opened, which is what makes the -N values in the body checkable for free. Measured 2026-09-03 on 1.10.0: exit 1 and the message naming the accepted set, mandatory yes 1 true disable optional no 0 false strict, with no connection line in the output. It runs against PROBE_PORT rather than a hardcoded one only so that a future variant of this probe can reach the scratch database; nothing listens is fine here, because the argument is rejected first." + }, + { + "sql": "USE master;", + "expect": { "error": 40508 }, + "why": "The reason the skill says the connection string must name the user database and never master: the database cannot be corrected at run time. Microsoft Learn states the number and the text, USE statement is not supported to switch between databases, and this repository's own container harness records the same behaviour. If a future engine build allowed USE, the must-have list in the body would be overcautious rather than wrong, and this probe going red is how anyone would find out." + }, + { + "sql": "SELECT encrypt_option FROM sys.dm_exec_connections WHERE session_id = @@SPID;", + "expect": { "rows": [["TRUE"]] }, + "why": "The correction the skill exists to make, asked of the engine rather than of the client: trusting a certificate does not turn encryption off. The lane connects with the image's ODBC Driver 18 sqlcmd, whose Encrypt default Learn records as changing to Yes as a declared breaking change, plus -C to accept the container's self-signed certificate. TRUE here means the session is encrypted while its certificate was never validated, which is exactly the distinction between Encrypt and TrustServerCertificate. Reasoned from Learn on 2026-09-03 and not yet measured on the container: if this comes back FALSE the reasoning is wrong, not the product, and the encryption paragraph in the body needs revisiting." + } + ] + }, + "triggering": { + "explicit": ["Use the connect-to-azure-sql skill"], + "implicit": [ + "how do I connect my app to Azure SQL Database", + "which driver should I use to talk to Azure SQL", + "add retry logic to my Azure SQL data access layer", + "what should my Azure SQL connection string contain", + "my first connection fails with error 40613 and the second one works" + ], + "routing": [ + "make the database layer resilient", + "the connection to Azure keeps dropping under load", + "should I set TrustServerCertificate to true to get past this certificate error", + "how many connections can my database take" + ], + "negative": [ + "install the ODBC driver in my Alpine Python image", + "set the max pool size in my Node.js app", + "create an Azure SQL database and open the firewall for my IP", + // No skill in this catalog owns this prompt. Azure SQL Managed Instance is a + // different product and nothing here covers it, so declining is the whole + // correct answer and there is no neighbour to hand it to. Measured + // 2026-09-20 across three models: claude-sonnet-5 named nothing, gpt-5.6-sol + // named this skill, which is why the description now states the boundary. + "connect to my Azure SQL Managed Instance", + "run Azure SQL locally in a container", + "what does error 40615 mean and how do I fix it" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema deleted file mode 120000 index 61a034f..0000000 --- a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema +++ /dev/null @@ -1 +0,0 @@ -../../../skills/design-azure-sql-schema \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/SKILL.md b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/SKILL.md new file mode 100644 index 0000000..b6c8bc4 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/SKILL.md @@ -0,0 +1,346 @@ +--- +name: design-azure-sql-schema +description: >- + Designs tables for Azure SQL Database so the first index or long value does not force a + rebuild. Covers index key limits in bytes, why Unicode sizing makes NVARCHAR(850) and + NVARCHAR(450) the real ceilings, collation decided once at CREATE DATABASE, the implicit + conversion that turns a lookup into a scan, identity gaps of a thousand after a restart, unique + columns that accept exactly one NULL. Use when creating or reviewing tables, porting a + PostgreSQL or MySQL schema to Azure SQL Database, choosing a key, a string length or a + collation, or reports a warning about maximum key length, an insert failing long after its + migration, a lookup that suddenly scans, identity values that jumped, or a duplicate key on + NULL. The engine rule under the mappers: ef-core-azure-sql and sqlalchemy-azure-sql own how + each expresses it, t-sql-correctness syntax, vector-search-azure-sql vector columns. +--- + +# Design a schema Azure SQL Database will not make you rebuild + +These are the schema decisions Azure SQL Database punishes **later**, not at `CREATE TABLE`. +Normalisation and naming are not here: they are the same everywhere. + +Everything below was measured on 2026-08-28 against a live engine where +`SERVERPROPERTY('EngineEdition')` returns `5` and `Edition` is `SQL Azure`. +Open [the measured statements](references/key-bytes-collation-and-identity-gaps.md) when a number +below disagrees with your engine, and before changing one. + +| | | +|---|---| +| The DDL | succeeds, sometimes with a warning nobody reads | +| The migration | succeeds | +| The failure | arrives on a row, a plan or a failover, weeks later, in production | + +**A syntax error costs minutes. Each item here costs a table rebuild, a collation rebuild or a +duplicated key.** + +## 1. The index key budget is in bytes, and Unicode spends two per character + +**1700 bytes for a nonclustered index key, 900 for a clustered one**, and a composite key spends +the **sum**. `NVARCHAR(n)` costs `2n` bytes, so: + +| Position | Widest safe Unicode column | Widest safe UTF-8 column | +|---|---|---| +| Nonclustered index or unique constraint | `NVARCHAR(850)` | `VARCHAR(1700)` | +| Clustered index or clustered primary key | `NVARCHAR(450)` | `VARCHAR(900)` | +| Anything not in a key | unbounded | unbounded | + +`NVARCHAR(450)` is the only length safe in **every** index position, which is why EF Core narrows +an indexed string to it. + +Over-shooting is a **warning**, message 1945 at severity 10. The index exists, the migration +reports success, short values insert fine: + +```text +Warning! The maximum key length for a nonclustered index is 1700 bytes. +The index 'ux_acct_email' has maximum length of 2000 bytes. +For some combination of large values, the insert/update operation will fail. +``` + +The failure waits for the first long row, and this one is an error: + +```text +Msg 1946, Level 16, State 3 +Operation failed. The index entry of length 1800 bytes for the index 'ux_acct_email' +exceeds the maximum length of 1700 bytes for nonclustered indexes. +``` + +**Consequence of being wrong: a write path that has worked for months starts rejecting exactly the +rows that matter,** and the fix is an `ALTER COLUMN` plus an index rebuild on a live table. + +- **`INCLUDE` columns do not count against the budget.** A covering index can include + `NVARCHAR(MAX)`. +- **A foreign key column must match the parent's length and collation exactly**, `Msg 1753` and + `Msg 1757`, so an oversized parent key propagates to every child. +- `NVARCHAR(MAX)`, `json` and `vector` cannot be key columns at all: `Msg 1919`, `Msg 1978`. + +## 2. Collation is chosen at CREATE DATABASE, and it decides two things + +A new database with no collation stated gets `SQL_Latin1_General_CP1_CI_AS`. + +`CI` means identifiers are not unique the way the design assumes: + +```sql +CREATE TABLE dbo.tokens (t NVARCHAR(64) NOT NULL PRIMARY KEY); +INSERT INTO dbo.tokens (t) VALUES (N'aB1x'); +INSERT INTO dbo.tokens (t) VALUES (N'Ab1X'); +-- Msg 2627, Violation of PRIMARY KEY constraint. The duplicate key value is (Ab1X). +``` + +**Consequence of being wrong: any column holding a case-significant token, API key, slug or short +link code collides at roughly the rate its alphabet implies**, as a user-visible insert failure. +The fix is per column: + +```sql +ALTER TABLE dbo.tokens ALTER COLUMN t NVARCHAR(64) COLLATE Latin1_General_CS_AS NOT NULL; +``` + +**State the collation in `CREATE DATABASE`, or accept the default deliberately.** There is no +server collation to inherit, and `ALTER DATABASE ... COLLATE` afterwards re-collates nothing that +already exists: it succeeds, changes only what is created next, and leaves you joining across a +collation conflict, `Msg 468`. The second half of the default's cost is section 3. + +## 3. The implicit conversion that costs the seek + +The common drivers bind a string parameter as Unicode by default. Under the **default** collation, +comparing that parameter to a `VARCHAR` column converts the **column**, and a converted column +cannot be sought. Measured over 20,000 rows with a unique index on `email`: + +| Column | Collation | Parameter | Plan | Logical reads | +|---|---|---|---|---| +| `VARCHAR(320)` | `SQL_Latin1_General_CP1_CI_AS` (the default) | `NVARCHAR` | **Index Scan** | **80** | +| `VARCHAR(320)` | `Latin1_General_CI_AS` | `NVARCHAR` | Index Seek | 2 | +| `VARCHAR(850)` | `Latin1_General_100_CI_AS_SC_UTF8` | `NVARCHAR` | Index Seek | 2 | +| `NVARCHAR(320)` | any | `NVARCHAR` | Index Seek | 2 | +| `VARCHAR(320)` | the default | `VARCHAR` | Index Seek | 2 | + +```text +|--Index Scan(OBJECT:([dbo].[users_v].[ux_v]), + WHERE:(CONVERT_IMPLICIT(nvarchar(320),[dbo].[users_v].[email],0)=[@p])) +``` + +Row two seeks because a Windows collation lets the optimizer build a range with +`GetRangeThroughConvert`. **A SQL collation cannot, and the default is a SQL collation.** + +**Consequence of being wrong: every lookup on that column reads the whole index instead of three +pages, forever, with no error and no warning.** Two designs avoid it, and one has to be chosen up +front: + +1. **`NVARCHAR` for anything a driver will bind a string to.** Correct, and it costs key bytes. +2. **A UTF-8 collation on `VARCHAR`** when the data is ASCII-dominant and the key is tight. + `VARCHAR(1700)` then fits a nonclustered key, and the seek survives because UTF-8 collations are + Windows collations. The cost is that `VARCHAR(n)` counts **bytes**. + +Do not mix. `VARCHAR` under the default collation plus an application binding Unicode is row one. + +## 4. Identity gaps are routine here, and the remembered fix does not exist + +Measured across one engine restart, which is what a planned failover looks like from inside: + +| Column | Last value before | First value after | Gap | +|---|---|---|---| +| `INT IDENTITY` | 3 | **1002** | 999 | +| `BIGINT IDENTITY` | 3 | **10002** | 9999 | +| `SEQUENCE`, default cache | 2 | **51** | 49 | +| `SEQUENCE ... NO CACHE` | 2 | 3 | none | +| `INT IDENTITY` with `IDENTITY_CACHE = OFF` | 3 | **4** | none | + +**Consequence of being wrong: any number a human reads or a regulator counts, an invoice number, a +ticket number, an order reference, develops thousand-wide holes, unrecoverably.** Nothing errors. + +```sql +-- The remembered remedy from a non-PaaS engine. Refused here. +DBCC TRACEON (272, -1); -- Msg 40518, DBCC command 'traceon' is not supported + +-- The lever that works. It survives a restart and costs a log write per value. +ALTER DATABASE SCOPED CONFIGURATION SET IDENTITY_CACHE = OFF; + +-- A human-facing number gets its own sequence, and START WITH is not optional: +-- omitted, the first value is -9223372036854775808. +CREATE SEQUENCE dbo.invoice_no AS BIGINT START WITH 1 INCREMENT BY 1 NO CACHE; +``` + +A surrogate key may be `IDENTITY`: gaps in a key nobody reads are free, and the cache is why +inserts are fast. A number a person reads is not a surrogate key. + +## 5. A unique index accepts exactly one NULL + +This one arrives from PostgreSQL habit, where every NULL is distinct. + +```sql +CREATE UNIQUE INDEX ux_code ON dbo.accounts (code); +INSERT INTO dbo.accounts (code) VALUES (NULL); -- 1 row +INSERT INTO dbo.accounts (code) VALUES (NULL); +-- Msg 2601, Cannot insert duplicate key row ... The duplicate key value is (). +-- Msg 2627 instead, where the constraint is a unique constraint rather than an index. + +-- The fix, verified to accept many NULL rows: +DROP INDEX ux_code ON dbo.accounts; +CREATE UNIQUE INDEX ux_code ON dbo.accounts (code) WHERE code IS NOT NULL; +``` + +**Consequence of being wrong: an optional unique field, an external account id, a nullable slug, +works for the first row that leaves it empty and rejects the second.** A model with two or three +optional unique fields is unusable here and correct on other engines. The session creating a +filtered index, and every session writing to the table, needs `QUOTED_IDENTIFIER ON`, else +`Msg 1934`. Drivers set it; a command line client may not. + +## 6. Where the json and vector types belong + +Both are storage decisions, not key decisions. + +| | `json` | `vector(n)` | +|---|---|---| +| Index key column | no, `Msg 1978` | no, `Msg 1978` | +| Primary key | no | no, `Msg 1919` | +| `INCLUDE` column | yes, verified | route to `vector-search-azure-sql` | +| Compared with `=` | no, `Msg 402` against `nvarchar` | route to `vector-search-azure-sql` | +| Ceiling | 2 GB | 1998 dimensions, `Msg 2717` above it | + +A `json` column is not a filter. Promote anything a query filters on **every time** into a real +column: + +```sql +ALTER TABLE dbo.doc ADD tenant AS CAST(JSON_VALUE(payload, '$.tenant') AS NVARCHAR(64)) PERSISTED; +CREATE INDEX ix_doc_tenant ON dbo.doc (tenant); +``` + +A `vector` column is a payload beside the key, never part of it. A `vector(1536)` occupies 6152 of +the 8060 bytes in a row, so keep the table narrow: put the embedding next to an integer key. + +## 7. Two boundaries a design cannot cross here + +```sql +-- No foreign key reaches another database, even one on the same server. +ALTER TABLE dbo.local_child ADD CONSTRAINT fk_x FOREIGN KEY (parent_id) + REFERENCES otherdb.dbo.some_parent (id); +-- Msg 40515, Reference to database and/or server name in 'otherdb.dbo.some_parent' +-- is not supported in this version of SQL Server. + +-- There is one filegroup, so a partition scheme maps everything to it. +SELECT name, type_desc FROM sys.filegroups; -- PRIMARY, and nothing else +CREATE PARTITION SCHEME ps AS PARTITION pf ALL TO ([PRIMARY]); +``` + +A shared-lookup-database design has no referential integrity here. Partitioning buys +manageability, never storage separation. + +## Check it worked + +Run the DDL through `sqlcmd` with `-m-1`, because message 1945 is **severity 10** and `-b` alone +does not report severity 10 at all, so a migration that emitted it still exits 0: + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build the `grep` below finds no message 1945 to report. `build-app-on-azure-sql` tells the two +builds apart in one table. + +```bash +export SQLCMDPASSWORD='' +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 \ + -i schema.sql -o schema.out +grep -nE 'Warning!|^Msg ' schema.out +``` + +Expect no output from `grep`. A `Warning!` line naming a key length is message 1945 and predicts +`Msg 1946` on a production insert. + +Then audit the schema the engine actually has, as `schema-audit.sql`: + +```sql +-- 1. Index keys over budget. Expect zero rows. +SELECT OBJECT_NAME(i.object_id) AS table_name, i.name AS index_name, i.type_desc, + SUM(CASE WHEN c.max_length = -1 THEN 0 ELSE c.max_length END) AS key_bytes, + CASE WHEN i.type = 1 THEN 900 ELSE 1700 END AS budget +FROM sys.indexes AS i +JOIN sys.index_columns AS ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id + AND ic.is_included_column = 0 +JOIN sys.columns AS c ON c.object_id = ic.object_id AND c.column_id = ic.column_id +WHERE i.type IN (1, 2) AND OBJECTPROPERTY(i.object_id, 'IsUserTable') = 1 +GROUP BY i.object_id, i.name, i.type, i.type_desc +HAVING SUM(CASE WHEN c.max_length = -1 THEN 0 ELSE c.max_length END) + > CASE WHEN i.type = 1 THEN 900 ELSE 1700 END; + +-- 2. Unique indexes over a nullable column with no filter: one NULL row each. Expect zero. +SELECT OBJECT_NAME(i.object_id) AS table_name, i.name AS index_name, c.name AS column_name +FROM sys.indexes AS i +JOIN sys.index_columns AS ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id + AND ic.is_included_column = 0 +JOIN sys.columns AS c ON c.object_id = ic.object_id AND c.column_id = ic.column_id +WHERE i.is_unique = 1 AND i.has_filter = 0 AND c.is_nullable = 1; + +-- 3. VARCHAR or CHAR key columns on a SQL collation: the lost seek. Expect zero rows. +SELECT DISTINCT OBJECT_NAME(c.object_id) AS table_name, c.name AS col, c.collation_name +FROM sys.columns AS c +JOIN sys.index_columns AS ic ON ic.object_id = c.object_id AND ic.column_id = c.column_id + AND ic.is_included_column = 0 +JOIN sys.types AS t ON t.user_type_id = c.user_type_id +WHERE t.name IN ('varchar', 'char') AND c.collation_name LIKE 'SQL[_]%'; + +-- 4. Sequences left on the type minimum because START WITH was omitted. Expect zero rows. +SELECT name, CAST(start_value AS BIGINT) AS start_value +FROM sys.sequences WHERE CAST(start_value AS BIGINT) < 0; + +-- 5. The two settings the design assumed. Read them, do not assume them. +SELECT DATABASEPROPERTYEX(DB_NAME(), 'Collation') AS db_collation, + (SELECT value FROM sys.database_scoped_configurations + WHERE name = 'IDENTITY_CACHE') AS identity_cache; +``` + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -W \ + -i schema-audit.sql -o schema-audit.out +``` + +Checks 1 to 4 return no rows in a clean schema. Check 5 is the one you read rather than pass: +`identity_cache` of `0` means gaps are off, `1` means an `INT IDENTITY` will jump by about a +thousand across the next reconfiguration. + +Last, confirm the seek survived on one real lookup: + +```sql +SET STATISTICS IO ON; +DECLARE @p NVARCHAR(320) = N'u9999@example.com'; +SELECT id FROM dbo.users WHERE email = @p; +SET STATISTICS IO OFF; +``` + +Expect single-digit logical reads that do not grow with the table. Reads in the tens over twenty +thousand rows is the implicit conversion in section 3, not a cold cache. + +## Do not + +- Do not read message 1945 as informational. It is the only notice given, and severity 10 means + neither `sqlcmd -b` nor most drivers raise it. +- Do not size a key column in characters. The limit is bytes, and Unicode doubles it. +- Do not use `VARCHAR` under the default collation for anything an application looks up by value. +- Do not add `LOWER()` or `UPPER()` around a column to fix a collation problem. That costs the + index too, and it is `t-sql-correctness`. +- Do not reach for trace flag 272. It is rejected with `Msg 40518`. The lever is + `IDENTITY_CACHE = OFF`. +- Do not put an invoice or ticket number, or anything an auditor reads, on a bare `IDENTITY`. +- Do not create a `SEQUENCE` without `START WITH`. +- Do not give a model two or three optional unique columns and expect PostgreSQL NULL semantics. +- Do not store a value in a `json` document and then filter on it. Promote it. +- Do not add a clustered index only because you believe this engine demands one. A heap was + created and took rows on 2026-08-28. + +## References + +- Open [the measured statements](references/key-bytes-collation-and-identity-gaps.md) when you need + the statement behind a number above, the two query plans, or the reproduction steps. +- Read [Maximum capacity specifications](https://learn.microsoft.com/sql/sql-server/maximum-capacity-specifications-for-sql-server) + before designing a wide composite key. +- Read [Collation and Unicode support](https://learn.microsoft.com/sql/relational-databases/collations/collation-and-unicode-support) + before choosing a database collation: SQL against Windows collations, and UTF-8. +- Read [json data type](https://learn.microsoft.com/sql/t-sql/data-types/json-data-type) before + putting a document in a column, for the index restrictions and the size limits. +- Read [Database scoped configurations](https://learn.microsoft.com/sql/t-sql/statements/alter-database-scoped-configuration-transact-sql) + before setting `IDENTITY_CACHE`, and + [T-SQL differences](https://learn.microsoft.com/azure/azure-sql/database/transact-sql-tsql-differences-sql-server) + when a statement is refused with 40510, 40515, 40517 or 40518. +- `ef-core-azure-sql` and `sqlalchemy-azure-sql`: how each mapper reaches these lengths and the + default it lands on. The other mapper skills are listed in `skill.spec.jsonc`. +- `t-sql-correctness` for query-level correctness, `t-sql-json-and-openjson` for querying a + document column, `vector-search-azure-sql` for the vector type and its indexing. +- `schema-migrations-safely` and `sql-database-projects` for applying a change to a live database, + and `provision-azure-sql-db` for creating the database and naming its collation. diff --git a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/references/key-bytes-collation-and-identity-gaps.md b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/references/key-bytes-collation-and-identity-gaps.md new file mode 100644 index 0000000..8621540 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/references/key-bytes-collation-and-identity-gaps.md @@ -0,0 +1,395 @@ +# Index key bytes, collation, identity gaps: the statements that produced every number + +Every number and message quoted in `SKILL.md` came from a run on **2026-08-28** against a live +engine reporting `EngineEdition` 5 and `Edition` `SQL Azure`, product version 12.0.2000.8, server +collation `SQL_Latin1_General_CP1_CI_AS`. Output is copied verbatim. + +## Contents + +- [How to reproduce](#how-to-reproduce) +- [Defaults a new database gets](#defaults-a-new-database-gets) +- [Index key limits in bytes](#index-key-limits-in-bytes) +- [What a key column may not be](#what-a-key-column-may-not-be) +- [Foreign keys must match exactly](#foreign-keys-must-match-exactly) +- [Collation and uniqueness](#collation-and-uniqueness) +- [The collation retrofit that does not work](#the-collation-retrofit-that-does-not-work) +- [Implicit conversion and the lost seek](#implicit-conversion-and-the-lost-seek) +- [Identity and sequence gaps across a restart](#identity-and-sequence-gaps-across-a-restart) +- [Unique indexes and NULL](#unique-indexes-and-null) +- [The json type](#the-json-type) +- [The vector type](#the-vector-type) +- [Boundaries](#boundaries) +- [What did not hold](#what-did-not-hold) + +## How to reproduce + +Run a local Azure SQL Database container, connect as an administrative login with a password taken +from the environment rather than written down, and confirm the engine before trusting anything +below: + +```sql +SELECT SERVERPROPERTY('EngineEdition') AS ee, -- 5 + SERVERPROPERTY('Edition') AS ed, -- SQL Azure + SERVERPROPERTY('Collation') AS servercoll; +``` + +Every statement below runs in a database created with `CREATE DATABASE schemalab;` and no other +options, with `SET QUOTED_IDENTIFIER ON` and `SET ANSI_NULLS ON`. + +## Defaults a new database gets + +```sql +CREATE DATABASE schemalab; +SELECT name, collation_name, is_read_committed_snapshot_on, compatibility_level +FROM sys.databases WHERE name = 'schemalab'; +``` + +```text +name collation_name is_read_committed_snapshot_on compatibility_level +schemalab SQL_Latin1_General_CP1_CI_AS 1 170 +``` + +Three things follow. The default collation is case-insensitive **and** it is a SQL collation, which +is the combination sections 2 and 3 of `SKILL.md` are about. Read committed snapshot isolation is +on by default. The compatibility level of a new database was 170. + +## Index key limits in bytes + +Each statement, and what the server returned. + +| Statement | Result | +|---|---| +| `CREATE UNIQUE INDEX ux ON t(NVARCHAR(1000) col)` | warning, 2000 bytes against 1700 | +| `CREATE UNIQUE INDEX ux ON t(NVARCHAR(851) col)` | warning, 1702 bytes against 1700 | +| `CREATE UNIQUE INDEX ux ON t(NVARCHAR(850) col)` | **clean** | +| `PRIMARY KEY CLUSTERED (NVARCHAR(451) col)` | warning, 902 bytes against 900 | +| `PRIMARY KEY CLUSTERED (NVARCHAR(450) col)` | **clean** | +| `CREATE UNIQUE INDEX ux ON t(a, b, c)`, three `NVARCHAR(300)` | warning, 1800 bytes against 1700 | +| `CREATE UNIQUE INDEX ux ON t(NVARCHAR(400) a) INCLUDE (NVARCHAR(4000), NVARCHAR(MAX))` | **clean** | +| `CREATE UNIQUE INDEX ux ON t(VARCHAR(1700) col COLLATE ..._UTF8)` | **clean** | +| `CREATE UNIQUE INDEX ux ON t(VARCHAR(1701) col COLLATE ..._UTF8)` | warning, 1701 bytes against 1700 | + +The warning, in full: + +```text +Warning! The maximum key length for a nonclustered index is 1700 bytes. +The index 'ux_acct_email' has maximum length of 2000 bytes. +For some combination of large values, the insert/update operation will fail. +``` + +`sys.messages` gives that message id **1945** at **severity 10**, so it is a warning and a client +that only surfaces errors shows nothing at all. + +The deferred failure, on the first row long enough to matter: + +```sql +INSERT INTO dbo.acct (email) VALUES (REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 900)); +``` + +```text +Msg 1946, Level 16, State 3, Server SQL Azure, Line 1 +Operation failed. The index entry of length 1800 bytes for the index 'ux_acct_email' +exceeds the maximum length of 1700 bytes for nonclustered indexes. +``` + +And where a column was narrowed rather than over-sized, the engine refuses instead of truncating: + +```sql +INSERT INTO dbo.n450 (slug) VALUES (REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 451)); +``` + +```text +Msg 2628, Level 16, State 1, Server SQL Azure, Line 1 +String or binary data would be truncated in table 'schemalab.dbo.n450', column 'slug'. +Truncated value: 'aaaaaaaa...'. +``` + +## What a key column may not be + +```text +CREATE INDEX ix_big ON dbo.lob_t (big); -- big is NVARCHAR(MAX) +Msg 1919 Column 'big' in table 'dbo.lob_t' is of a type that is invalid for use as a key column + in an index. + +CREATE INDEX ix_doc ON dbo.doc (payload); -- payload is json +Msg 1978 Column 'payload' in table 'dbo.doc' is of a type that is invalid for use as a key column + in an index or statistics. + +CREATE TABLE dbo.emb2 (v VECTOR(4) NOT NULL PRIMARY KEY); +Msg 1919 Column 'v' in table 'emb2' is of a type that is invalid for use as a key column + in an index. +``` + +A fixed-width row over the page limit is refused at create time: + +```text +CREATE TABLE dbo.wide_fixed (a NCHAR(4000), b NCHAR(1000)); +Msg 1701 Creating or altering table 'wide_fixed' failed because the minimum row size would be + 10007, including 7 bytes of internal overhead. This exceeds the maximum allowable table + row size of 8060 bytes. +``` + +The variable-width equivalent, `NVARCHAR(4000)` plus `NVARCHAR(1000)`, was created without a +warning, because those columns are eligible for row overflow. + +## Foreign keys must match exactly + +```text +-- parent NVARCHAR(64), child NVARCHAR(128) +Msg 1753 Column 'dbo.p2.k' is not the same length or scale as referencing column 'c2.k' in + foreign key 'FK__c2__k__0F624AF8'. Columns participating in a foreign key relationship + must be defined with the same length and scale. + +-- parent COLLATE Latin1_General_CS_AS, child on the database default +Msg 1757 Column 'dbo.p3.k' is not of same collation as referencing column 'c3.k' in + foreign key 'FK__c3__k__14270015'. +``` + +A parent key of `NVARCHAR(900)`, 1800 bytes, raised the 1945 warning on its own primary key and the +child column had to match it, so an oversized key propagates rather than staying local. + +## Collation and uniqueness + +```sql +CREATE TABLE dbo.tok_ci (t NVARCHAR(64) NOT NULL PRIMARY KEY); +CREATE TABLE dbo.tok_cs (t NVARCHAR(64) COLLATE Latin1_General_CS_AS NOT NULL PRIMARY KEY); + +INSERT INTO dbo.tok_cs (t) VALUES (N'aB1x'), (N'Ab1X'); -- 2 rows +INSERT INTO dbo.tok_ci (t) VALUES (N'aB1x'); -- 1 row +INSERT INTO dbo.tok_ci (t) VALUES (N'Ab1X'); +``` + +```text +Msg 2627, Level 14, State 1, Server SQL Azure, Line 2 +Violation of PRIMARY KEY constraint 'PK__tok_ci__3BD01999B8939C9B'. +Cannot insert duplicate key in object 'dbo.tok_ci'. The duplicate key value is (Ab1X). +``` + +Final counts: `tok_cs` 2 rows, `tok_ci` 1 row. + +## The collation retrofit that does not work + +```sql +-- in a fresh database on the default collation +CREATE TABLE dbo.old_t (c VARCHAR(50) NOT NULL); +-- sys.columns.collation_name for old_t.c: SQL_Latin1_General_CP1_CI_AS + +ALTER DATABASE collab2 COLLATE Latin1_General_CS_AS; -- succeeds + +-- sys.columns.collation_name for old_t.c: SQL_Latin1_General_CP1_CI_AS, UNCHANGED +CREATE TABLE dbo.new_t (c VARCHAR(50) NOT NULL); +-- sys.columns.collation_name for new_t.c: Latin1_General_CS_AS + +SELECT COUNT(*) FROM dbo.old_t o JOIN dbo.new_t n ON o.c = n.c; +``` + +```text +Msg 468, Level 16, State 9, Server SQL Azure, Line 2 +Cannot resolve the collation conflict between "Latin1_General_CS_AS" and +"SQL_Latin1_General_CP1_CI_AS" in the equal to operation. +``` + +`CREATE DATABASE collab COLLATE Latin1_General_100_CI_AS_SC_UTF8;` was accepted, so stating the +collation at creation is available and is the only version of this decision that works cleanly. +`sys.fn_helpcollations()` returned 1585 collations whose name ends in `_UTF8`. + +## Implicit conversion and the lost seek + +Three tables, 20,000 identical rows of the form `u@example.com`, each with a unique index on +`email`. + +```sql +CREATE TABLE dbo.users_v (id INT IDENTITY PRIMARY KEY, email VARCHAR(320) NOT NULL); +CREATE TABLE dbo.users_w (id INT IDENTITY PRIMARY KEY, + email VARCHAR(320) COLLATE Latin1_General_CI_AS NOT NULL); +CREATE TABLE dbo.users_n (id INT IDENTITY PRIMARY KEY, email NVARCHAR(320) NOT NULL); +``` + +```sql +DECLARE @p NVARCHAR(320) = N'u12345@example.com'; +SELECT id FROM dbo.users_v WHERE email = @p; +``` + +`SET STATISTICS IO ON` reported, for one row returned: + +| Table | Column type and collation | Scan count | Logical reads | +|---|---|---|---| +| `users_v` | `VARCHAR`, `SQL_Latin1_General_CP1_CI_AS` | 1 | **80** | +| `users_w` | `VARCHAR`, `Latin1_General_CI_AS` | 1 | 2 | +| `users_n` | `NVARCHAR` | 0 | 2 | +| `users_v` with a `VARCHAR` parameter | `VARCHAR`, default collation | 0 | 2 | +| `u8` | `VARCHAR`, `Latin1_General_100_CI_AS_SC_UTF8` | 0 | 2 | + +`SET SHOWPLAN_TEXT ON` for the same three statements: + +```text +-- users_v, VARCHAR under the default SQL collation +|--Index Scan(OBJECT:([dbo].[users_v].[ux_v]), + WHERE:(CONVERT_IMPLICIT(nvarchar(320),[dbo].[users_v].[email],0)=[@p])) + +-- users_w, VARCHAR under a Windows collation +|--Compute Scalar(DEFINE:(([Expr1005],[Expr1006],[Expr1004])=GetRangeThroughConvert(...))) +|--Index Seek(OBJECT:([dbo].[users_w].[ux_w]), + SEEK:([email] > [Expr1005] AND [email] < [Expr1006]), ...) + +-- users_n, NVARCHAR +|--Index Seek(OBJECT:([dbo].[users_n].[ux_n]), SEEK:([email]=[@p]) ORDERED FORWARD) +``` + +The conversion is applied to the **column** in the first plan, which is what removes the seek. In +the second it is applied to the parameter and a range is derived instead. The UTF-8 collation +behaved like the Windows collation, because it is one. + +## Identity and sequence gaps across a restart + +Setup, then a full engine restart, which is what a planned reconfiguration looks like from inside +the database. + +```sql +CREATE TABLE dbo.inv_int (id INT IDENTITY(1,1) PRIMARY KEY, note NVARCHAR(20)); +CREATE TABLE dbo.inv_big (id BIGINT IDENTITY(1,1) PRIMARY KEY, note NVARCHAR(20)); +CREATE SEQUENCE dbo.seq_cached AS BIGINT START WITH 1 INCREMENT BY 1; +CREATE SEQUENCE dbo.seq_nocache AS BIGINT START WITH 1 INCREMENT BY 1 NO CACHE; +-- three inserts into each table, two values from each sequence +``` + +| | Last value before restart | First value after | +|---|---|---| +| `INT IDENTITY` | 3 | **1002** | +| `BIGINT IDENTITY` | 3 | **10002** | +| `SEQUENCE`, default cache | 2 | **51** | +| `SEQUENCE ... NO CACHE` | 2 | 3 | + +The fix an agent recalls from a non-PaaS engine is refused: + +```text +DBCC TRACEON (272, -1); +Msg 40518, Level 16, State 5, Server SQL Azure, Line 2 +DBCC command 'traceon' is not supported in this version of SQL Server. +``` + +The lever that does work, and a second restart to prove it: + +```sql +ALTER DATABASE SCOPED CONFIGURATION SET IDENTITY_CACHE = OFF; +-- three inserts, MAX(id) = 3, restart, one insert +``` + +`MAX(id)` afterwards was **4**, and `sys.database_scoped_configurations` still reported +`IDENTITY_CACHE = 0`, so the setting survives the restart. + +A sequence with no `START WITH`: + +```sql +CREATE SEQUENCE dbo.seq_nostart AS BIGINT; +SELECT NEXT VALUE FOR dbo.seq_nostart; -- -9223372036854775808 +``` + +`sys.sequences` reported `start_value` and `minimum_value` both `-9223372036854775808`. + +## Unique indexes and NULL + +```sql +CREATE UNIQUE INDEX ux_code ON dbo.nulltest (code); +INSERT INTO dbo.nulltest (code) VALUES (NULL); -- 1 row +INSERT INTO dbo.nulltest (code) VALUES (NULL); +``` + +```text +Msg 2601, Level 14, State 1, Server SQL Azure, Line 1 +Cannot insert duplicate key row in object 'dbo.nulltest' with unique index 'ux_code'. +The duplicate key value is (). +``` + +The same violation through a unique **constraint** rather than an index reports `Msg 2627`. + +The filtered index accepts many: + +```sql +CREATE UNIQUE INDEX ux_code2 ON dbo.nulltest2 (code) WHERE code IS NOT NULL; +INSERT INTO dbo.nulltest2 (code) VALUES (NULL), (NULL); +SELECT COUNT(*) FROM dbo.nulltest2; -- 5 +``` + +Creating it under a client that had not set `QUOTED_IDENTIFIER ON` failed with `Msg 1934`, naming +`QUOTED_IDENTIFIER` directly. Drivers set it; a command line client may not. + +## The json type + +```sql +CREATE TABLE dbo.doc (id INT IDENTITY PRIMARY KEY, payload JSON NULL); +``` + +`sys.columns` reports type `json` with `max_length` of `-1`. + +```text +CREATE INDEX ix_doc ON dbo.doc (payload); +Msg 1978 ... invalid for use as a key column in an index or statistics. + +SELECT COUNT(*) FROM dbo.doc WHERE payload = N'{"tenant":"acme","n":1}'; +Msg 402 The data types json and nvarchar are incompatible in the equal to operator. +``` + +The promotion pattern, both statements clean: + +```sql +ALTER TABLE dbo.doc ADD tenant AS CAST(JSON_VALUE(payload, '$.tenant') AS NVARCHAR(64)) PERSISTED; +CREATE INDEX ix_doc_tenant ON dbo.doc (tenant); +CREATE INDEX ix_doc_inc ON dbo.doc (tenant) INCLUDE (payload); -- json as an included column +``` + +Microsoft Learn adds that the type is generally available on Azure SQL Database, is available under +**all** database compatibility levels, stores up to 2 GB, and permits a `json` column as an +included column and in the `WHERE` clause of a filtered index. Read the +[json data type](https://learn.microsoft.com/sql/t-sql/data-types/json-data-type) page for the full +size table rather than copying it here. + +## The vector type + +```text +CREATE TABLE dbo.emb3 (v VECTOR(1999)); +Msg 2717 The size (1999) given to the column 'v' exceeds the maximum allowed (1998). +``` + +`VECTOR(1998)` was accepted. `sys.columns` reports `max_length` 6152 for a `VECTOR(1536)`, which is +`1536 * 4 + 8`. A table carrying a `VECTOR(1536)` beside two `NVARCHAR(1000)` columns allocated a +`ROW_OVERFLOW_DATA` unit, so the vector is pushed off-row once the row is wide. + +## Boundaries + +```text +ALTER TABLE dbo.local_child ADD CONSTRAINT fk_x FOREIGN KEY (parent_id) + REFERENCES otherdb.dbo.some_parent (id); +Msg 40515, Level 15, State 1, Server SQL Azure, Line 16 +Reference to database and/or server name in 'otherdb.dbo.some_parent' is not supported +in this version of SQL Server. +``` + +Both databases existed on the same container, so this is the database boundary being enforced and +not a missing object. + +```sql +SELECT name, type_desc FROM sys.filegroups; -- PRIMARY, ROWS_FILEGROUP. One row. +CREATE PARTITION SCHEME ps AS PARTITION pf TO ([PRIMARY], [SECONDARY]); -- Msg 208, invalid object name 'SECONDARY' +CREATE PARTITION SCHEME ps2 AS PARTITION pf ALL TO ([PRIMARY]); -- succeeds +``` + +A table with no clustered index was created and accepted rows, so a heap is permitted. The belief +that this engine requires a clustered index on every table is out of date. + +## What did not hold + +Recorded so the next author does not re-derive it. + +- **The catalog note said "identity against sequences" as though the choice were a style + preference.** It is not. The measured gap is 999 for `INT` and 9999 for `BIGINT`, and a default + `SEQUENCE` gapped by 49, so a sequence is only the safer choice when it is declared `NO CACHE`. + A sequence with the default cache is no better than an identity column. +- **`ALTER DATABASE ... COLLATE` reads like a fix and is not one.** It was expected either to fail + or to work. It did neither: it succeeded and changed nothing that mattered, which is the worse of + the two outcomes. +- **`DATABASEPROPERTYEX(db, 'CatalogCollation')` returned `NULL`** on this container rather than a + collation name, so no claim about catalog collation is made in `SKILL.md`. +- **`ALTER DATABASE ... SET COMPATIBILITY_LEVEL = 150` did not take effect** on this container: the + database stayed at 170. Any claim tying a type to a compatibility level should be taken from + Microsoft Learn rather than measured here. diff --git a/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/skill.spec.jsonc new file mode 100644 index 0000000..5292880 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/design-azure-sql-schema/skill.spec.jsonc @@ -0,0 +1,345 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "design-azure-sql-schema", + "domain": "schema-and-orm", + "value": [ + "corrects-a-wrong-assumption", + "multi-step-with-silent-failures" + ], + "correction": "Asked to design a schema for Azure SQL Database, an agent sizes string columns in characters, takes the default collation without noticing what it decides, and treats an identity column as a counter that increments by one. Every failure that follows is deferred rather than immediate, which is why review does not catch them. Measured on 2026-08-28 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure': a unique index on NVARCHAR(1000) is created with message 1945 at severity 10, a warning most clients never surface, and the first genuinely long value fails months later with Msg 1946, because the nonclustered key limit of 1700 bytes and the clustered limit of 900 bytes are byte counts and Unicode spends two per character, making NVARCHAR(850) and NVARCHAR(450) the real ceilings. The default collation SQL_Latin1_General_CP1_CI_AS is case-insensitive, so a table of slugs or API keys rejects 'Ab1X' beside 'aB1x' with Msg 2627, and it is also a SQL collation, which is the one case where a VARCHAR column compared to the Unicode parameter every mainstream driver binds loses its index seek: measured 80 logical reads against 2 over 20,000 rows, with CONVERT_IMPLICIT applied to the column inside an Index Scan, while the identical column under a Windows or UTF-8 collation seeks. Identity is wrong in the same deferred way. The remembered remedy, trace flag 272, is rejected here with Msg 40518, and measured across a single engine restart an INT IDENTITY jumped from 3 to 1002, a BIGINT IDENTITY from 3 to 10002 and a default-cached SEQUENCE from 2 to 51, so any invoice or ticket number built on a bare identity column develops thousand-wide holes on routine reconfiguration, silently and unrecoverably, unless the design uses SEQUENCE ... NO CACHE or ALTER DATABASE SCOPED CONFIGURATION SET IDENTITY_CACHE = OFF.", + "value_probe": { + "gate": "The answer finds the three deferred failures before any of them happens, being an index key limit counted in bytes rather than characters, a default collation that ignores case under a key the application treats as case sensitive, and a numbering source that develops large gaps on its own, and it changes the definition rather than only describing the risks.", + "task": "I am about to add a table on Azure SQL Database and I would like a review before I run it in production.\n\n```sql\nCREATE TABLE dbo.api_client (\n invoice_no INT IDENTITY(1,1) PRIMARY KEY,\n client_slug NVARCHAR(1000) NOT NULL,\n created_at DATETIME2 NOT NULL\n);\nCREATE UNIQUE INDEX ux_api_client_slug ON dbo.api_client (client_slug);\n```\n\nThe slug is generated by our application and it is case sensitive by design, so 'Ab1X' and 'aB1x' are two different clients and both have to be able to exist. The invoice number is printed on documents customers keep, and our finance team expects the numbers to run consecutively with no holes.\n\nThe database was created with the portal defaults and nobody changed anything about it. I ran this against a test database and it completed with no error at all. What is going to go wrong later, and what should I change before this goes live?", + "states": [ + "The index key limit is counted in bytes and a Unicode character costs two of them, so the declared length is about double what the index can actually hold.", + "The index statement is accepted now and the failure appears only when someone inserts a value long enough to exceed the limit, which may be months later.", + "The default collation ignores case, so two slugs differing only in case collide on the unique index and the second one is rejected.", + "Identity values are handed out from a cache, so a restart or a reconfiguration of the service leaves a gap of hundreds or thousands with no rows deleted and nothing to recover.", + "A consecutive number needs a source that is explicitly not cached, or the caching turned off at the database, rather than a plain identity column." + ], + "refutes": [ + "The index key limit is a character count, so a 1000 character Unicode column is comfortably inside it and the unique index is fine as written.", + "The default collation on a new Azure SQL Database distinguishes case, so two slugs differing only in case will be stored as two separate rows.", + "An identity column produces consecutive values, so the invoice numbers cannot develop gaps unless rows are deleted or a transaction is rolled back.", + "A server level trace flag can be enabled on Azure SQL Database to stop the identity column skipping values.", + "The statement completing with no error on the test database means the index definition is valid and there is nothing further to check about its width." + ] + }, + "posture": [ + "read", + "write", + "inspect" + ], + "maturity": "ga", + "applies_to": [ + "azure-sql-db", + "azure-sql-db-container" + ], + "related": [ + "ef-core-azure-sql", + "prisma-azure-sql", + "sqlalchemy-azure-sql", + "t-sql-correctness", + "t-sql-json-and-openjson", + "vector-search-azure-sql", + "schema-migrations-safely", + "sql-database-projects", + "provision-azure-sql-db", + "diagnose-slow-query", + "build-app-on-azure-sql" + ], + "validation": { + "target": "both", + "setup": [ + "a database created with no explicit collation, so it carries the default", + "an administrative connection able to create databases, tables and indexes", + "the ability to restart or fail over the engine, for the identity cache assertions" + ], + "assert": [ + "no CREATE INDEX or CREATE TABLE in the generated schema emits message 1945, so every key column total is at or under 1700 bytes nonclustered and 900 bytes clustered", + "every string column participating in an index key, a unique constraint or a foreign key carries an explicit length rather than a mapper default", + "the database collation is stated in CREATE DATABASE, or a comment records that the default was chosen deliberately", + "every column holding a case-significant token, code or key carries an explicit case-sensitive collation", + "no table combines a VARCHAR column on a SQL collation with an application that binds Unicode string parameters, and a plan taken for one such lookup shows an Index Seek rather than an Index Scan with CONVERT_IMPLICIT on the column", + "no human-facing sequential number is generated by a bare IDENTITY column", + "every CREATE SEQUENCE states START WITH", + "every nullable unique column uses a filtered unique index, or the design states that one NULL row is the intent", + "no json or vector column appears in a key, a GROUP BY, a DISTINCT or an equality predicate", + "no foreign key names a second database" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_keylen (id INT IDENTITY PRIMARY KEY, email NVARCHAR(1000) NULL);\nCREATE UNIQUE INDEX ux_probe_keylen ON dbo.probe_keylen (email);\nINSERT INTO dbo.probe_keylen (email) VALUES (REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 900));", + "expect": { + "error": 1946 + }, + "why": "The measurement the whole skill turns on, made executable. The CREATE UNIQUE INDEX on 2000 bytes of key succeeds and emits only message 1945 at severity 10, which most clients never surface, so this batch reaches the INSERT. The 900 character value is 1800 bytes and is refused with Msg 1946. Measured 2026-08-28. The failure arriving on the row rather than on the DDL is the point." + }, + { + "sql": "CREATE TABLE dbo.probe_keywarn (id INT IDENTITY PRIMARY KEY, email NVARCHAR(1000) NULL);\nCREATE UNIQUE INDEX ux_probe_keywarn ON dbo.probe_keywarn (email);", + "expect": { + "warning": 1945 + }, + "why": "The other half of the pair above, and the half nobody sees. The same DDL that sets up Msg 1946 is asserted here on its own: the index over a 2000 byte key is CREATED, the statement succeeds, and the only notice given is message 1945 at severity 10. Measured 2026-08-28, text \"Warning! The maximum key length for a nonclustered index is 1700 bytes\", which Microsoft Learn quotes verbatim on Index JSON data, and sys.messages gives the id 1945 at severity 10. Asserting it needs sqlcmd -m-1, because without that a severity 10 message prints no number and -b never exits non-zero below severity 11, which is exactly why a migration emitting this reports success. Left unasserted, only the deferred error was measured and the moment the mistake is made was not." + }, + { + "sql": "CREATE TABLE dbo.probe_keysum (a NVARCHAR(300) NOT NULL, b NVARCHAR(300) NOT NULL, c NVARCHAR(300) NOT NULL);\nCREATE UNIQUE INDEX ux_probe_keysum ON dbo.probe_keysum (a, b, c);", + "expect": { + "warning": 1945 + }, + "why": "A composite key spends the SUM of its columns, which section 1 states and nothing measured. No column here is anywhere near the limit at 600 bytes each, and the three together are 1800 bytes, so the same message 1945 is emitted and the index is still created. Measured 2026-08-28. This is the shape a real design reaches the warning through, because each column looks harmless on its own and only the key is over budget." + }, + { + "sql": "CREATE TABLE dbo.probe_ceiling (email NVARCHAR(850) NOT NULL);\nCREATE UNIQUE INDEX ux_probe_ceiling ON dbo.probe_ceiling (email);\nINSERT INTO dbo.probe_ceiling (email) VALUES (REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 850));\nSELECT COUNT(*) AS rows_accepted FROM dbo.probe_ceiling;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The other half of the same claim: NVARCHAR(850) is 1700 bytes exactly, so it is the widest Unicode column a nonclustered key can carry, and a value using every one of those characters is accepted rather than merely tolerated. Measured 2026-08-28, where 851 raised message 1945 and 850 was clean. Without this probe the table of ceilings in section 1 is only half tested." + }, + { + "sql": "CREATE TABLE dbo.probe_inc (id INT IDENTITY PRIMARY KEY, k NVARCHAR(400) NOT NULL, blob NVARCHAR(MAX) NULL);\nCREATE UNIQUE INDEX ux_probe_inc ON dbo.probe_inc (k) INCLUDE (blob);", + "expect": { + "succeeds": true + }, + "why": "INCLUDE columns do not count against the 1700 byte key budget, and NVARCHAR(MAX) is legal there even though it can never be a key column. Measured 2026-08-28. This is the escape hatch the skill offers whenever a design wants a wide column covered, so it is worth proving rather than asserting." + }, + { + "sql": "CREATE TABLE dbo.probe_lob (id INT NOT NULL, big NVARCHAR(MAX) NULL);\nCREATE INDEX ix_probe_lob ON dbo.probe_lob (big);", + "expect": { + "error": 1919 + }, + "why": "NVARCHAR(MAX) cannot be an index key column at all, which is a different refusal from being over the byte budget: it is rejected at create time rather than warned about. Measured 2026-08-28, Msg 1919, column is of a type that is invalid for use as a key column in an index." + }, + { + "sql": "CREATE TABLE dbo.probe_json (id INT IDENTITY PRIMARY KEY, payload JSON NULL);\nCREATE INDEX ix_probe_json ON dbo.probe_json (payload);", + "expect": { + "error": 1978 + }, + "why": "A json column is a storage decision, not a key decision, and the engine says so at create time. Measured 2026-08-28, Msg 1978, invalid for use as a key column in an index or statistics. The distinct number from 1919 is why section 6 names both." + }, + { + "sql": "CREATE TABLE dbo.probe_json_eq (id INT IDENTITY PRIMARY KEY, payload JSON NULL);\nSELECT COUNT(*) FROM dbo.probe_json_eq WHERE payload = N'{\"a\":1}';", + "expect": { + "error": 402 + }, + "why": "A json column cannot be compared with equality against a string, so any design that treats the document as a filterable value fails at the first query rather than at the schema review. Measured 2026-08-28, Msg 402, the data types json and nvarchar are incompatible in the equal to operator." + }, + { + "sql": "CREATE TABLE dbo.probe_doc (id INT IDENTITY PRIMARY KEY, payload JSON NULL);\nALTER TABLE dbo.probe_doc ADD tenant AS CAST(JSON_VALUE(payload, '$.tenant') AS NVARCHAR(64)) PERSISTED;\nCREATE INDEX ix_probe_doc_tenant ON dbo.probe_doc (tenant);\nINSERT INTO dbo.probe_doc (payload) VALUES (N'{\"tenant\":\"acme\"}');\nSELECT tenant FROM dbo.probe_doc;", + "expect": { + "rows": [ + [ + "acme" + ] + ] + }, + "why": "The remedy the skill offers for the previous probe, run end to end: promote the value out of the document into a PERSISTED computed column and index that. Measured 2026-08-28. Asserting the promoted value rather than plain success proves the computed column is actually populated from the document and not merely accepted by the parser." + }, + { + "sql": "CREATE TABLE dbo.probe_tok_ci (t NVARCHAR(64) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_tok_ci (t) VALUES (N'aB1x');\nINSERT INTO dbo.probe_tok_ci (t) VALUES (N'Ab1X');", + "expect": { + "error": 2627 + }, + "why": "The default collation of a new database is SQL_Latin1_General_CP1_CI_AS, and CI means two case-different tokens are the same key. Measured 2026-08-28, Msg 2627. The collation is stated on the column rather than inherited so the probe measures case insensitivity itself and not whatever collation the scratch database happens to carry." + }, + { + "sql": "CREATE TABLE dbo.probe_tok_cs (t NVARCHAR(64) COLLATE Latin1_General_CS_AS NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_tok_cs (t) VALUES (N'aB1x'), (N'Ab1X');\nSELECT COUNT(*) AS rows_accepted FROM dbo.probe_tok_cs;", + "expect": { + "rows": [ + [ + 2 + ] + ] + }, + "why": "The fix for the previous probe, and the reason the skill says the fix is per column rather than per database: the same two values are two distinct keys under an explicit case-sensitive collation. Measured 2026-08-28, both rows accepted." + }, + { + "sql": "CREATE TABLE dbo.probe_null (id INT IDENTITY PRIMARY KEY, code NVARCHAR(64) NULL);\nCREATE UNIQUE INDEX ux_probe_null ON dbo.probe_null (code);\nINSERT INTO dbo.probe_null (code) VALUES (NULL);\nINSERT INTO dbo.probe_null (code) VALUES (NULL);", + "expect": { + "error": 2601 + }, + "why": "A unique index here treats NULL as a value, so it accepts exactly one NULL row and refuses the second. This is the belief an agent carries over from PostgreSQL, where every NULL is distinct. Measured 2026-08-28, Msg 2601, the duplicate key value is ()." + }, + { + "sql": "CREATE TABLE dbo.probe_null_f (id INT IDENTITY PRIMARY KEY, code NVARCHAR(64) NULL);\nCREATE UNIQUE INDEX ux_probe_null_f ON dbo.probe_null_f (code) WHERE code IS NOT NULL;\nINSERT INTO dbo.probe_null_f (code) VALUES (NULL), (NULL), (N'x');\nSELECT COUNT(*) AS rows_accepted FROM dbo.probe_null_f;", + "expect": { + "rows": [ + [ + 3 + ] + ] + }, + "why": "The filtered unique index the skill prescribes, proved to accept many NULL rows while still enforcing uniqueness on the values that are present. Measured 2026-08-28. It also exercises the QUOTED_IDENTIFIER requirement: a session without it fails to create this index with Msg 1934, so a green result says the runner's session is set correctly too." + }, + { + "sql": "DBCC TRACEON (272, -1);", + "expect": { + "error": 40518 + }, + "why": "The remedy an agent recalls for identity gaps from a non-PaaS engine, and the measured proof it does not exist here. Measured 2026-08-28, Msg 40518. Microsoft Learn lists 40518 as \"DBCC command '%.*ls' is not supported in this version of SQL Server\" and lists trace flags among the T-SQL not supported in Azure SQL Database, so the number and the refusal are both sourced." + }, + { + "sql": "CREATE TABLE dbo.probe_fk_child (id INT NOT NULL PRIMARY KEY, parent_id INT NOT NULL);\nALTER TABLE dbo.probe_fk_child ADD CONSTRAINT fk_probe_x FOREIGN KEY (parent_id)\n REFERENCES otherdb.dbo.some_parent (id);", + "expect": { + "error": 40515 + }, + "why": "A foreign key cannot reach another database, so a shared-lookup-database design has no referential integrity here. Measured 2026-08-28 with both databases present on the same server, which is why this is the database boundary and not a missing object. Microsoft Learn gives 40515 as \"Reference to database and/or server name in '%.*ls' is not supported in this version of SQL Server\"." + }, + { + "sql": "CREATE TABLE dbo.probe_p (k NVARCHAR(64) NOT NULL PRIMARY KEY);\nCREATE TABLE dbo.probe_c (id INT NOT NULL PRIMARY KEY, k NVARCHAR(128) NOT NULL);\nALTER TABLE dbo.probe_c ADD CONSTRAINT fk_probe_len FOREIGN KEY (k) REFERENCES dbo.probe_p (k);", + "expect": { + "error": 1753 + }, + "why": "A child key column must match the parent's length exactly, which is how an oversized parent key propagates through a schema instead of staying local. Measured 2026-08-28, Msg 1753, columns participating in a foreign key relationship must be defined with the same length and scale." + }, + { + "sql": "CREATE SEQUENCE dbo.probe_seq_nostart AS BIGINT;\nSELECT CASE WHEN CAST(start_value AS BIGINT) = CAST(minimum_value AS BIGINT) THEN 1 ELSE 0 END AS starts_at_type_minimum\nFROM sys.sequences WHERE name = 'probe_seq_nostart';", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "A CREATE SEQUENCE with no START WITH does not start at 1, it starts at the minimum of its type, which for BIGINT is -9223372036854775808. Measured 2026-08-28. The comparison is made against the catalog's own minimum_value rather than a literal, because the literal -9223372036854775808 is parsed as numeric rather than bigint and would make the probe about T-SQL literal typing instead." + }, + { + "sql": "CREATE TABLE dbo.probe_heap (id INT NOT NULL, note NVARCHAR(20) NULL);\nINSERT INTO dbo.probe_heap (id, note) VALUES (1, N'a'), (2, N'b');\nSELECT (SELECT COUNT(*) FROM dbo.probe_heap) AS rows_accepted,\n (SELECT COUNT(*) FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.probe_heap') AND type = 1) AS clustered_indexes;", + "expect": { + "rows": [ + [ + 2, + 0 + ] + ] + }, + "why": "The correction in the Do not list, made executable: a table with no clustered index is created and takes rows. The belief that this engine requires a clustered index on every table is out of date. Measured 2026-08-28. Two counts are asserted together so the probe cannot pass on a table that quietly acquired a clustered index from a primary key." + } + , + { + "sql": "CREATE TABLE dbo.probe_d_ci (s nvarchar(20) COLLATE Latin1_General_CI_AS);\nCREATE TABLE dbo.probe_d_bin (s nvarchar(20) COLLATE Latin1_General_BIN2);\nGO\nSELECT a.s FROM dbo.probe_d_ci AS a JOIN dbo.probe_d_bin AS b ON a.s = b.s;", + "expect": { + "error": 468 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A join across two collations is refused rather than resolved, which is the second half of what the database default costs. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_p20 (k nvarchar(20) NOT NULL PRIMARY KEY);\nCREATE TABLE dbo.probe_d_c50 (k nvarchar(50) NOT NULL, CONSTRAINT fk_probe_d_len FOREIGN KEY (k) REFERENCES dbo.probe_d_p20(k));", + "expect": { + "error": 1753 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A foreign key column must match the parent's length exactly, so an oversized parent key propagates to every child rather than being narrowed at the child. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_pc (k nvarchar(20) COLLATE Latin1_General_CI_AS NOT NULL PRIMARY KEY);\nCREATE TABLE dbo.probe_d_cc (k nvarchar(20) COLLATE Latin1_General_BIN2 NOT NULL, CONSTRAINT fk_probe_d_coll FOREIGN KEY (k) REFERENCES dbo.probe_d_pc(k));", + "expect": { + "error": 1757 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The same rule for collation, which is the one that bites when a single column was given a collation by hand. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_nvmax (s nvarchar(max) NOT NULL PRIMARY KEY);", + "expect": { + "error": 1919 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. nvarchar(max) cannot be a key column. Note the number: this is 1919 and the json and vector cases below are 1978, which is why the skill prints both. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_jkey (id int NOT NULL PRIMARY KEY, payload json);\nGO\nCREATE INDEX probe_d_jkey_ix ON dbo.probe_d_jkey (payload);", + "expect": { + "error": 1978 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A json column cannot be an index key, the row this skill's storage table asserts. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_vkey (id int NOT NULL PRIMARY KEY, v vector(3));\nGO\nCREATE INDEX probe_d_vkey_ix ON dbo.probe_d_vkey (v);", + "expect": { + "error": 1978 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And neither can a vector column, the other half of the same row. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_dim (v vector(1999));", + "expect": { + "error": 2717 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The 1998 dimension ceiling, asserted here as well as in vector-search-azure-sql because this skill is where the choice of column type is made. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "DBCC TRACEON (272, -1);", + "expect": { + "error": 40518 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Trace flag 272, the identity cache lever every answer from a boxed engine reaches for, is not available here, which is why the skill routes to a sequence instead. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT * FROM probe_d_otherdb.dbo.some_parent;", + "expect": { + "error": 40515 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A three part name across databases is refused, which is what makes a cross database foreign key impossible rather than merely discouraged. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_uidx (id int IDENTITY, s nvarchar(20) NULL);\nCREATE UNIQUE INDEX probe_d_uidx_ix ON dbo.probe_d_uidx (s);\nGO\nINSERT INTO dbo.probe_d_uidx (s) VALUES (NULL);\nINSERT INTO dbo.probe_d_uidx (s) VALUES (NULL);", + "expect": { + "error": 2601 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A unique index treats two NULLs as duplicates, which is the behaviour that makes an optional unique field unusable without a filtered index. The number differs from the constraint case below, and the skill says so. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_ucon (s nvarchar(20) NOT NULL CONSTRAINT uq_probe_d UNIQUE);\nGO\nINSERT INTO dbo.probe_d_ucon (s) VALUES (N'a');\nINSERT INTO dbo.probe_d_ucon (s) VALUES (N'a');", + "expect": { + "error": 2627 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The same collision through a unique constraint reports a different number, which is the distinction the skill draws so a log line can be read back to its cause. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_d_filt (id int NOT NULL PRIMARY KEY, slug nvarchar(50) NULL);\nCREATE UNIQUE INDEX probe_d_filt_ix ON dbo.probe_d_filt (slug) WHERE slug IS NOT NULL;\nGO\nSET QUOTED_IDENTIFIER OFF;\nINSERT INTO dbo.probe_d_filt (id, slug) VALUES (1, N'a');", + "expect": { + "error": 1934 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The filtered index is the fix for the NULL problem above, and it puts a requirement on every session that writes to the table. Drivers set QUOTED_IDENTIFIER; a command line client may not, which is how this reaches production as an intermittent write failure. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Use the design-azure-sql-schema skill before you create these tables", + "Design the schema for this application on Azure SQL Database", + "Review my CREATE TABLE statements before I run the first migration", + "What collation should this Azure SQL database use", + "How long should I make the string columns that are in my indexes", + "Should this primary key be an IDENTITY column or a SEQUENCE" + ], + "implicit": [ + "Warning! The maximum key length for a nonclustered index is 1700 bytes", + "my migration ran fine and now inserts are failing on the unique index", + "our invoice numbers jumped by about a thousand overnight and nobody deleted anything", + "why can I only have one row with a NULL in this unique column", + "two users signed up with the same email in different cases and the second one was rejected", + "where should I put the embeddings, in the same table as the documents", + "can I put a foreign key from this table to a lookup table in our other database", + "should this column be varchar or nvarchar" + ], + "routing": [ + "model these entities as tables for our new service", + "port this PostgreSQL schema onto Azure SQL Database", + "review this schema before it goes to production" + ], + "negative": [ + "Configure UseAzureSql and the retry execution strategy on my DbContext", + "Set the right length on a Prisma String field so migrate stops warning", + "Map a SQLAlchemy model so it stops creating non Unicode columns", + "Write me a paginated query for Azure SQL", + "How do I get the id of the row I just inserted", + "Write a similarity search over my embeddings with VECTOR_DISTANCE", + "Apply this column rename to the live database without downtime", + "Create the Azure SQL database and its firewall rule", + "Why is this particular query slow" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors deleted file mode 120000 index 579ec97..0000000 --- a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors +++ /dev/null @@ -1 +0,0 @@ -../../../skills/diagnose-connection-errors \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/SKILL.md b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/SKILL.md new file mode 100644 index 0000000..3f84585 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/SKILL.md @@ -0,0 +1,218 @@ +--- +name: diagnose-connection-errors +description: >- + Diagnoses an Azure SQL Database connection refused before a credential was evaluated, reading + the error number to name the layer that refused: the IP firewall (40615), a virtual network + rule (40914), public network access disabled (47073, 42101), the gateway declining a server + name or login format (40532, 40531), a TLS version under the server minimum (47072), plus the + transport, pre-login, timeout and certificate failures that carry no number. Use when someone + pastes an error naming an IP address, a firewall, the gateway, a certificate chain, the + pre-login handshake or a bare timeout, before anyone edits a connection string: none of these + is a connection-string problem. Not what the server decided after reading the credential: a + login refused, or a valid login with no database user, is entra-id-auth's; a database asking + for a retry while it resumes is connect-to-azure-sql's. +--- + +# Diagnose an Azure SQL Database connection error + +**Read the number, then read its severity.** The number names the layer that refused, and the +severity says what kind of layer it was. Microsoft Learn defines the levels: **11** is an object or +entity that does not exist, **14** is security, **16** is user-correctable configuration, **17** is +the service out of some resource, **20 and above** is fatal and usually takes the connection with +it, and **10** is informational and is *converted to severity 0 before it reaches the calling +application*. That last one is why the worst-explained connection failures look like they carry no +number at all. + +Sourced from Microsoft Learn on 2026-09-03, and checked against sqlcmd 1.10.0 and Azure CLI 2.90.0. + +## Step 1: make the number visible + +`-b` sets a non-zero exit only at severity 11 and above. Without `-m-1` a severity 10 message prints +its text with no `Msg` number, which is the class 47072, 47073 and 42101 belong to. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build those three numbers never appear at all, and `sys.messages` is the way to get them. `build-app-on-azure-sql` tells the two +builds apart in one table. + +```bash +sqlcmd -S your-server.database.windows.net,1433 -d your-database -U your-login -P "$SQL_PASSWORD" \ + -N -l 30 -b -m-1 -Q "SELECT 1" +``` + +`-N` requests an encrypted connection and `-l 30` is the connection timeout Learn recommends as a +floor. Then resolve the number on the engine rather than from memory, with the +catalog view Learn publishes for exactly this: + +```sql +SELECT message_id, severity, is_event_logged, [text] +FROM sys.messages +WHERE language_id = 1033 AND message_id IN (4060, 18456, 40532, 40613, 40615) +ORDER BY message_id; +``` + +## The severity column is the diagnosis + +| Number | Sev | What refused, and what the severity says about it | First move | +|---|---|---|---| +| `40615` | 16 | No IP rule covers the address the server saw | Add the IP rule, then allow five minutes | +| `40914` | not sourced | The client subnet has service endpoints and the server has no virtual network rule | Add the virtual network rule for the subnet | +| `47073` | 10 | Public network access is Disable, and severity 10 means the client may see no number | Use the private endpoint, or set public access to selected networks | +| `42101` | 10 | A firewall rule change while public network access is Disable | Set public access to selected networks first | +| `47072` | 10 | The client offered a TLS version below the server minimum | Raise the client to TLS 1.2 or higher | +| `40532` | 11 | 11 is "the given object does not exist", and the object is the **server**, not the login | Check the server name and login format | +| `40531` | 11 | The gateway could not determine the server name from the connection | Send `username@servername`, or fix the first DNS segment | +| `18456` | 14 | Security: the login was evaluated and refused | Named here, answered by `entra-id-auth` | +| `4060` | 11 | The object that does not exist is the **user in that database** | Named here, answered by `entra-id-auth` | +| `40613` | 17 | The service is out of something. On a paused serverless or free database it is the expected first answer | `connect-to-azure-sql` owns the retry doctrine | +| `10928` / `10929` | 16 | A limit was reached. Resource ID 1 is workers, 2 is sessions | See `diagnose-resource-pressure` | +| `40544` | 20 | Size quota, and 20 is fatal, so the connection is gone and retrying on it cannot work | Free space or scale up | +| `40501` / `49918` / `40197` | 16 | Busy or reconfiguring. Transient | Retry with backoff | + +`18456`, `4060` and `40613` keep a row so a pasted number resolves, and nothing more. + +## 47072, 47073 and 42101 are severity 10, and Learn disagrees with itself + +The connectivity settings page prints them as top-level errors, for example `Error 47073`. The +error number tables list all three at **severity 10** with text beginning `Reason:`, the shape of a +sub-message attached to a login failure rather than a standalone error. Both are Microsoft Learn and +they do not agree. + +Take the tables as operative: they explain the symptom. What a client sees was measured on +2026-09-08, by setting public network access to Disabled on a live logical server and connecting: + +```output +mssql: login error: Reason: An instance-specific error occurred while establishing a connection to +SQL Server. Connection was denied because Deny Public Network Access is set to Yes. For more +information, see https://go.microsoft.com/fwlink/?linkid=2323206. +``` + +**It is not a timeout, and it is not silent.** The refusal is immediate and go-sqlcmd 1.10.0 prints +the severity 10 sub-message text in full, remedy and documentation link included. What it does not +print is the bare number, so a developer searching their own terminal output for `47073` finds +nothing while the reason is on the screen in front of them. Read the reason, not the number, and look +the number up in `sys.messages` on a connection that works if you need it for a ticket. + +## 40615 and 40914 are the firewall, and no connection string fixes them + +The address to allow is **the address the server observed**, which network address translation +usually makes different from the client's configured address. `40615` names it in the message, so +read it there rather than guessing. + +```bash +az sql server firewall-rule list -g your-resource-group -s your-server -o table +az sql server firewall-rule create -g your-resource-group -s your-server \ + -n AllowDevWorkstation --start-ip-address 192.168.1.0 --end-ip-address 192.168.1.255 +``` + +Two things that go wrong here: + +- **Changes take up to five minutes.** A rule that looks correct and still fails may simply be + young. `DBCC FLUSHAUTHCACHE;` refreshes it sooner, but it runs **in the user database**, never in + the logical `master` where the rules physically live, needs `KILL DATABASE CONNECTION` or the + admin account, and needs a connection that already works. It is a tool for the second database, + not the one locking you out. +- **A rule from `0.0.0.0` to `0.0.0.0` is not a wildcard for the internet.** It is the documented + `AllowAllWindowsAzureIps` special case, and Microsoft warns it admits Azure resources in *other + customers' subscriptions*. Prefer a virtual network rule or a private endpoint. + +## 40532 is not 18456 and it is not 4060 + +All three read as "login failed" and are routinely answered as one thing. Telling them apart is the +diagnosis this skill exists to make, because `40532` is fixed by correcting the connection target +and the other two never are. The severity table above splits them: `40532` is the gateway refusing +before any database was reached, `18456` is the login refused at severity 14, and `4060` is a valid +login with no user in the database it asked for. + +Check the login format before the credential. `40531` documents the rule: the server name must be +the first segment of the DNS name, and some libraries do not send it at all, in which case it travels +as `username@servername`. If both are sent they must match. + +Azure SQL Database withholds the reason from the client and there is no server error log to read the +state from, so do not promise a diagnosis the platform does not expose. Narrow it by what changed: a +new server, a new password, a new database name, a new deployment slot. + +One more check when it fails on some client networks only. A hosts file or a private DNS zone +pinning the server name to a retired gateway address produces persistent 18456, 40532 and 40615 +failures, because the gateway validates the name it was reached by, and logins sent straight to an +IP address fail by design. + +## Public network access and the minimum TLS version + +Both are server properties and both refuse at severity 10, so read them rather than infer them: + +```bash +az sql server show -n your-server -g your-resource-group \ + --query "{publicNetworkAccess:publicNetworkAccess, minimalTlsVersion:minimalTlsVersion}" +``` + +TLS 1.0 and 1.1 are retired, the lowest minimum Azure SQL Database supports is 1.2, and **once a +minimum is enforced it cannot be reverted to the default**. Enforcing 1.3 breaks clients whose +driver or operating system does not support it. + +## Errors with no number, and the certificate case + +When the remote host terminates the TCP connection the client library raises the error, so the SQL +number is `0` and the text belongs to the TCP provider. The one to recognise names the **pre-login +handshake** and a TCP provider error 0. Microsoft's guidance is a **fixed number of +retry attempts** before declaring failure. This is also the usual shape of a firewall or network +appliance dropping the connection mid-stream. + +**The certificate failure is a different animal.** `The certificate chain was issued by an authority +that is not trusted` means the client does not trust the public root the service certificate chains +to, or is not reaching the service at all. The roots to have are DigiCert Global Root G2, Microsoft +ECC Root Certificate Authority 2017 and Microsoft RSA Root Certificate Authority 2017. + +Test it with encryption on and no trust override. If this succeeds, the chain validates and the +certificate is not your problem: + +```bash +sqlcmd -S your-server.database.windows.net,1433 -U your-login -P "$SQL_PASSWORD" \ + -N -l 30 -b -m-1 -Q "SELECT 1" +``` + +Do not resolve it with `-C` or `TrustServerCertificate=true` outside local development. That +disables the validation the error is reporting and leaves the connection encrypted but +unauthenticated, which is the attack the check exists to prevent. Fix the trust store instead. + +## Local container differences + +The `40xxx` family is the gateway, the IP firewall and the service's resource governance, and the +local Azure SQL Database container has none of the three, so an agent chasing a firewall rule to +explain a local failure is chasing something that is not there. What remains locally is the +numberless class above, which behaves the same in both places. A `40xxx` failure in the cloud had no +local equivalent to fail on, so a clean local run is not evidence either way. + +## Check it worked + +Run the same command that failed, `-b -m-1` still on, and check three things in order. + +```bash +sqlcmd -S your-server.database.windows.net,1433 -d your-database -U your-login -P "$SQL_PASSWORD" \ + -N -l 30 -b -m-1 -Q "SELECT DB_NAME() AS db, SUSER_SNAME() AS login_name;" +echo "exit=$?" +``` + +- **Exit 0 and one row** naming the database and login you expected. Exit 0 with no row means the + statement never ran. +- **A different number is progress, not failure.** `40615` becoming `18456` means the firewall fix + worked and the credential is the next problem, which is `entra-id-auth`'s. +- **The same number after a firewall change** is most likely the five minute propagation delay, not + a wrong rule. Confirm the rule with `az sql server firewall-rule list` before changing anything + else. + +## Before reporting a diagnosis + +- [ ] The number and its severity were both read before any file was edited +- [ ] `-m-1` was on, so a severity 10 refusal could not hide as a timeout +- [ ] A `40xxx` error was not blamed on the connection string +- [ ] `40532` was told apart from `18456` and `4060`, not merged into "login failed" +- [ ] A certificate error was not answered by trusting the server certificate +- [ ] Nothing past the credential was answered here + +## References + +- [references/error-codes.md](references/error-codes.md): open it when the number is not in the + table above, or when the exact documented wording matters, because it carries the verbatim text, + the severity and the Microsoft Learn page each number came from. diff --git a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/references/error-codes.md b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/references/error-codes.md new file mode 100644 index 0000000..ca8fc25 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/references/error-codes.md @@ -0,0 +1,265 @@ +# Azure SQL Database connection error codes + +Every entry was checked against Microsoft Learn on 2026-09-03. Message text is quoted as the +documentation writes it, including the `%.*ls` and `{0}` placeholders the engine fills in. Severity +comes from the Database Engine error number tables, which are the same text `sys.messages` holds. + +**Severity is half the diagnosis.** Learn's levels, in the range that reaches a connection: **10** +informational, and *converted to severity 0 before the error reaches the calling application*; **11** +the given object or entity does not exist; **14** security, such as permission denied; **16** a +general user-correctable error; **17** out of a resource or past a system-administrator limit; +**20 to 24** fatal, the task terminates and the connection usually goes with it. `TRY...CATCH` +catches above 10 and below the levels that kill the connection, so a severity 10 message reaches +neither a `CATCH` block nor, without sqlcmd's `-m-1`, a `Msg` number on the console. + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [Network and gateway: nothing reached the login](#network-and-gateway-nothing-reached-the-login) +- [Authentication: the login was rejected](#authentication-the-login-was-rejected) +- [Transient: retry is the fix](#transient-retry-is-the-fix) +- [Resource governance: a limit was reached](#resource-governance-a-limit-was-reached) +- [TLS and certificates](#tls-and-certificates) +- [Errors with no SQL number](#errors-with-no-sql-number) +- [Sources](#sources) + +## How to use this file + +Find the number. Read the **Sev** and **Means** columns, not the message: the message is written for +the person who configured the server, and it names the component that refused rather than the one +that needs changing. + +To resolve a number this file does not carry, ask the engine. Microsoft Learn publishes this query +as the way to read the same table these pages are generated from: + +```sql +SELECT message_id AS Error, + severity AS Severity, + [Event Logged] = CASE is_event_logged WHEN 0 THEN 'No' ELSE 'Yes' END, + [text] AS [Description] +FROM sys.messages +WHERE language_id = 1033 /* 1033 is US English */ +ORDER BY message_id; +``` + +A `Sev` of `not sourced` means the message text came from a product page that prints no severity and +the number was not in the error number tables. Do not fill one in from memory; run the query above. + +Numbers marked **retry** are documented transient faults. Everything else is a configuration or +credential problem, and retrying it just fails more slowly. + +The index is deliberately wider than the skill. Everything that rejects a connection **before the +login is evaluated** is answered here. `18456`, `4060` and `40613` sit the other side of that line +and are indexed only so a number can be identified and then routed, `18456` and `4060` to +`entra-id-auth` and `40613` to `connect-to-azure-sql`. Identification, not doctrine. + +## Network and gateway: nothing reached the login + +| Number | Sev | Message text | Means | Fix | +|---|---|---|---|---| +| `40615` | 16 | `Cannot open server '%.*ls' requested by the login. Client with IP address '%.*ls' is not allowed to access the server. To enable access, use the Azure Management Portal or run sp_set_firewall_rule on the master database to create a firewall rule for this IP address or address range. It may take up to five minutes for this change to take effect.` | No IP firewall rule covers the address the server saw | Add a server-level or database-level IP rule for that address | +| `40914` | not sourced | `Cannot open server '[server-name]' requested by the login. Client is not allowed to access the server.` | The client subnet has service endpoints but the server has no virtual network rule for it | Add a virtual network rule for the subnet | +| `47073` | 10 | Two documented wordings, see below | Public network access is set to Disable | Connect through the private endpoint, or set public access to selected networks | +| `42101` | 10 | Two documented wordings, see below | An attempt to change firewall rules while public network access is disabled | Set public network access to selected networks, then change the rules | +| `40611` / `40614` | 16 | `Azure SQL Database supports a maximum of 256 firewall rules.` / `Start IP address of firewall rule cannot exceed End IP address.` | The rule itself was refused, not the connection | Consolidate ranges, or swap the two addresses | +| `5` | not sourced | Cannot connect to the server | Outbound TCP 1433 is blocked between the client and the internet | Open outbound 1433 on every firewall in the path | +| `26`, `40`, `10053` | not sourced | `A network-related or instance-specific error occurred`, `A transport-level error has occurred when receiving results from the server` | The server was not found or the connection was aborted | Check the fully qualified server name and port 1433, then work the connectivity steps in order | + +**Two Microsoft Learn pages disagree about 47073 and 42101, and the difference matters.** The +connectivity settings page prints them as top-level errors: + +```output +Error 47073 +An instance-specific error occurred while establishing a connection to SQL Server. +The public network interface on this server is not accessible. +To connect to this server, use the Private Endpoint from inside your virtual network. + +Error 42101 +Unable to create or modify firewall rules when public network interface for the server is disabled. +To manage server or database level firewall rules, please enable the public network interface. +``` + +The error number table lists both at **severity 10** with different text, beginning `Reason:`, which +is the shape of a sub-message attached to a login failure rather than a standalone error: +`47073 Reason: An instance-specific error occurred while establishing a connection to SQL Server. +Connection was denied because Deny Public Network Access is set to Yes.` and `42101 Reason: Unable +to create or update firewall rules because Deny Public Network Access is set to Yes.` + +Both are Microsoft Learn. Take the severity from the table, because it explains the symptom: a +severity 10 message is converted to severity 0 on the way to the client, so a developer whose public +network access was disabled usually reports an unexplained failure or a timeout and never sees the +number. `sqlcmd -m-1` is what makes it visible. + +Two things that make firewall diagnosis go wrong: + +- **Propagation.** Security setting changes have a documented latency of five minutes. Run + `DBCC FLUSHAUTHCACHE` in the **user** database to refresh sooner. It does not apply to the logical + `master`, which is where the logins and firewall rules physically live, and it needs the + `KILL DATABASE CONNECTION` permission or the admin account. It also needs a connection that + already works, so it cannot rescue the connection the firewall is refusing. +- **Address translation.** The address the server enforces against is the one it observed, which + network address translation frequently makes different from the client's configured address. + +The `0.0.0.0` to `0.0.0.0` rule is a documented special case meaning "allow Azure services", named +`AllowAllWindowsAzureIps`. Microsoft's own warning is that it admits any Azure resource with +outbound connectivity, **including resources in other customers' subscriptions**. It is not a +wildcard for the public internet, and it is not a substitute for a virtual network rule or a +private endpoint. + +## Authentication: the login was rejected + +Past that line, apart from `40532` and `40531`. `18456` and `4060` are here to be recognised and +handed to `entra-id-auth`, which owns logins, database users and identities. Their Fix column +records what that skill will do, so the number can be told apart from `40532`. + +| Number | Sev | Message text | Means | Fix | +|---|---|---|---|---| +| `18456` | 14 | `Login failed for user '%.*ls'.%.*ls%.*ls` | The login does not exist, is disabled, or the password is wrong | Confirm the login exists in `sys.sql_logins` and is not disabled, then the password | +| `18456` with `` | 14 | `Login failed for user ''.` | A Microsoft Entra token was presented and accepted, but no matching principal exists in the database | Create the database user for that identity. Route to `entra-id-auth` | +| `4060` | 11 | `Cannot open database "%.*ls" requested by the login. The login failed.` | The login is valid; it has no user in the database it asked for, or the database name is wrong | Create the database user, or correct the database name | +| `40532` | 11 | `Cannot open server "%.*ls" requested by the login. The login failed.` | The gateway rejected the login before any database was reached | Check the server name and the login format the client sends | +| `40531` | 11 | `Server name cannot be determined. It must appear as the first segment of the server's dns name (servername.%.*ls). Some libraries do not send the server name, in which case the server name must be included as part of the user name (username@servername). In addition, if both formats are used, the server names must match.` | The gateway could not work out which server the login was for | Send `username@servername`, or correct the first DNS segment | +| `40608` | 10 | `This session has been assigned a tracing ID of '%.*ls'. Provide this tracing ID to customer support when you need assistance.` | The tracing ID that arrives appended to a login failure. Severity 10, so it is informational, not the error | Quote it in a support case, do not diagnose from it | + +**`18456` carries its real reason as a separate severity 10 message that Azure SQL Database does not +send to the client.** The reason strings have their own numbers in the 18300 to 18400 and 47000 +ranges: `18301 Could not find a login matching the name provided`, `18307 Password did not match +that for the login provided`, `18395 Authentication was successful, but database was not found on +this logical server`, `18397 Unable to retrieve database firewall rules`, `47070 Azure Active +Directory only authentication is enabled`. Severity 10 is converted to 0 before the calling +application sees it, which is why the client message hides the reason. Do not promise a diagnosis +the platform does not expose. + +Diagnose by difference instead: what changed between the last working connection and this one. + +Microsoft Learn groups `18456`, `40532` and `40615` together for one specific failure shape: +persistent login failures **isolated to particular client networks** while the service is healthy. +The documented cause there is a DNS override, a hosts file entry or a static record pinning the +server name to a retired gateway address. The gateway validates the name it was reached by, and +login attempts sent directly to an IP address fail by design. + +## Transient: retry is the fix + +All of these are documented transient faults. The documented retry shape is a 5 second delay +before the first retry, growing exponentially to a maximum of 60 seconds. + +| Number | Sev | Message text | Means | +|---|---|---|---| +| `40613` | 17 | `Database '%.*ls' on server '%.*ls' is not currently available. Please retry the connection later. If the problem persists, contact customer support, and provide them the session tracing ID of '%.*ls'.` | The database is resuming, is reconfiguring, or already has a dedicated administrator connection. On a paused serverless database this is the documented response to the first connection attempt. Identification only: the retry doctrine that answers it is `connect-to-azure-sql`'s, and a 40613 that survives a retry on a database that is neither serverless nor free is the dedicated administrator connection instead | +| `40197` | not sourced | `The service has encountered an error processing your request. Please try again. Error code %d.` | A failover or upgrade. The embedded code (40020, 40143, 40166, 40540 are examples) is the detail worth logging | +| `40501` | not sourced | `The service is currently busy. Retry the request after 10 seconds. Incident ID: %ls. Code: %d.` | Engine throttling, meaning resource limits are being exceeded | +| `49918` | not sourced | `Cannot process request. Not enough resources to process request. The service is currently busy. Please retry the request later.` | Control plane is out of capacity for the request | +| `49919` | not sourced | `Cannot process create or update request. Too many create or update operations in progress for subscription "%ld".` | Too many concurrent provisioning operations on the subscription | +| `49920` | not sourced | `Cannot process request. Too many operations in progress for subscription "%ld".` | As above, for operations generally | +| `4221` | not sourced | `Login to read-secondary failed due to long wait on 'HADR_DATABASE_WAIT_FOR_TRANSITION_TO_VERSIONING'.` | A read replica is not yet serving logins | + +Severity 17 on `40613` reads correctly against the definition: the service ran out of something or +passed a limit, which is what a paused database resuming is. It is not a defect and it is not +damage. + +**Serverless auto-pause and resume are `connect-to-azure-sql`'s, not this file's.** The one fact +needed to identify a `40613` here: any login attempt triggers the resume, and resume latency is +generally on the order of one minute. + +## Resource governance: a limit was reached + +| Number | Sev | Message text | Means | +|---|---|---|---| +| `10928` | 16 | `Resource ID : %d. The %ls limit for the database is %d and has been reached.` | Resource ID 1 is **workers**, Resource ID 2 is **sessions**. The message says "request limit" for backward compatibility, and the limit actually reached is workers | +| `10929` | 16 | `Resource ID : %d. The %ls minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d. However, the server is currently too busy to support %ls greater than %d for this database.` | The database is over its guaranteed share while the server is saturated | +| `40544` | 20 | `The database '%.*ls' has reached its size quota. Partition or delete data, drop indexes, or consult the documentation for possible resolutions.` | Storage, not compute. Severity 20 is fatal, so the connection is gone and retrying on it cannot work | +| `40549` | 16 | `Session is terminated because you have a long running transaction. Try shortening your transaction.` | One transaction held too long. Batch the work | +| `40550` | not sourced | `The session has been terminated because it has acquired too many locks. Try reading or modifying fewer rows in a single transaction.` | Lock count, not lock time | +| `40551` | not sourced | Terminated for excessive tempdb usage | | +| `40552` | not sourced | Terminated for excessive transaction log space usage | | +| `40553` | not sourced | Terminated for excessive memory usage | | + +The 10928 wording is the trap. Workers can far outnumber requests when the degree of parallelism is +above one, so the limit arrives sooner than the word "request" suggests. Sizing the pool against +sessions rather than workers is the standing mistake: `connect-to-azure-sql` owns the preventive +half, `diagnose-resource-pressure` the reactive half. + +## TLS and certificates + +| Number | Sev | Message text | Means | +|---|---|---|---| +| `47072` | 10 | `Reason: Login failed due to client TLS version being less than minimal TLS version allowed by the server.` in the error number table; `Login failed with invalid TLS version` on the connectivity settings page | The client offered a TLS version below the server's configured minimum. The same two-wordings problem as 47073 | +| `18399` | 10 | `Reason: Unsecured connection to the database is disallowed. The secured connection string should be used instead. [Database: '%.*ls']` | The client asked for an unencrypted connection. Turn encryption on rather than turning validation off | + +Current state, and it has moved recently enough to be worth re-checking rather than recalling: TLS +1.0 and 1.1 are retired and no longer available, so the lowest minimum Azure SQL Database supports +is 1.2; the default allows 1.2 and above and **once a minimum is enforced it cannot be reverted to +the default**; and enforcing 1.3 breaks clients whose driver or operating system does not support +it. Read the current value with +`az sql server show -n your-server -g your-resource-group --query "minimalTlsVersion"`. + +The certificate failure has no SQL number and reads, in the .NET stack, as `The certificate chain +was issued by an authority that is not trusted`. It became common when driver defaults changed to +encrypt with full certificate validation, so a client that previously never validated now does. + +The roots to trust are **DigiCert Global Root G2**, **Microsoft ECC Root Certificate Authority +2017** and **Microsoft RSA Root Certificate Authority 2017**. A driver using the operating system +certificate store on a maintained machine already has them. Turning validation off makes the error +stop and makes the connection unauthenticated: never outside local development. + +## Errors with no SQL number + +The client library raises these, so the SQL number is `0` and the text belongs to the TCP provider. +Documented examples: + +``` +A connection was successfully established with the server, but then an error occurred during the +pre-login handshake. (provider: TCP Provider, error: 0 - An existing connection was forcibly +closed by the remote host.) + +A transport-level error has occurred when sending the request to the server. (provider: TCP +Provider, error: 0 - An existing connection was forcibly closed by the remote host.) +``` + +The same sentence also appears with `during the login process` in place of `during the pre-login +handshake`, and means the same thing. + +Causes range from the database being briefly unavailable to a firewall or network appliance in the +path. The documented guidance is a **fixed number of retries** before treating them as permanent. + +Timeouts sit in the same bucket. `Connection Timeout Expired` while consuming the pre-login +handshake acknowledgement, and the plain `Timeout expired`, both mean the client could not reach the +server, which a blocked port, a missing firewall rule or disabled public network access all produce. +The recommended connection timeout is at least 30 seconds, which is `sqlcmd -l 30`. + +To tell connectivity from a slow query, read the stack: frames opening a connection point at +connectivity, frames executing a command point at the query. + +## Sources + +All Microsoft Learn. The error number tables and the severity definitions were read on 2026-09-03; +the rest on 2026-08-27. + +- Database Engine error severities: `/sql/relational-databases/errors-events/database-engine-error-severities` + (every level, and the rule that severity 10 is converted to 0 before the calling application sees + it, plus how `TRY...CATCH` treats each range) +- Database Engine events and errors, the range tables carrying severity and the verbatim + `sys.messages` text, under + `/sql/relational-databases/errors-events/database-engine-events-and-errors-`: + `10000-to-10999` (10928, 10929), `18000-to-18999` (18456 and the `Reason:` strings 18301, 18307, + 18395, 18397, 18399), `31000-to-41399` (40531, 40532, 40544, 40549, 40608, 40611, 40613, 40614, + 40615) and `41400-to-49999` (42101, 47070, 47072, 47073) +- Troubleshoot connectivity issues: `/azure/azure-sql/database/troubleshoot-common-errors-issues` + (transient fault table, resource governance table, DNS override guidance, network termination + errors, timeout errors) +- Virtual network endpoints and rules: `/azure/azure-sql/database/vnet-service-endpoint-rule-overview` + (verbatim 40914 and 40615 message text, description and resolution) +- IP firewall rules: `/azure/azure-sql/database/firewall-configure` (rule levels, the five minute + latency, `DBCC FLUSHAUTHCACHE`, the `0.0.0.0` rule and its warning) +- Connectivity settings: `/azure/azure-sql/database/connectivity-settings` (verbatim 47073, 42101 + and 47072 text, minimum TLS version state and defaults) +- Serverless auto-pause and auto-resume: `/azure/azure-sql/database/serverless-tier-auto-pause-resume` + (40613 on first connect to a paused database, resume latency, auto-pause triggers) +- Serverless compute tier: `/azure/azure-sql/database/serverless-tier-overview` (auto-pause delay + default and range) +- Certificate rotation: `/azure/azure-sql/updates/ssl-root-certificate-expiring` (the three roots) +- Error 18456: `/sql/relational-databases/errors-events/mssqlserver-18456-database-engine-error` +- `DBCC FLUSHAUTHCACHE`: `/sql/t-sql/database-console-commands/dbcc-flushauthcache-transact-sql` + (it does not apply to the logical `master`, it needs `KILL DATABASE CONNECTION` or the admin + account, and it clears cached Microsoft Entra group membership as well as logins and firewall + rules) diff --git a/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/skill.spec.jsonc new file mode 100644 index 0000000..9d1192a --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/diagnose-connection-errors/skill.spec.jsonc @@ -0,0 +1,141 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "diagnose-connection-errors", + "domain": "drivers-and-connectivity", + "value": ["changes-too-fast-to-memorize", "multi-step-with-silent-failures"], + "correction": "Shown a failed connection to Azure SQL Database, an agent reads every error as a credential or connection-string problem and starts editing the connection string, when the error number already names the layer that rejected it and that layer is usually in front of the login rather than behind it: 40615 is a missing IP firewall rule and 40914 the same refusal from a virtual network rule, neither of which any connection string can fix, 42101 and 47073 mean public network access is turned off, 40532 is the gateway refusing before a database was reached at all and is not the login failure it reads as, 47072 is a TLS version the server will not accept, and a certificate-chain failure is a client trust store problem that trusting the server certificate hides rather than fixes. The failures carrying no SQL number at all are the ones most often given up on as unknown, when their message text is the whole diagnosis.", + "value_probe": { + "task": "Our platform team moved one of our services onto new networking last night. Since then every attempt from that service to reach its database in Azure fails. The same code, the same image and the same configuration still work from a developer laptop and from our older environment.\n\nThis is the whole of what the application logs:\n\n```\nA connection was successfully established with the server, but then an error occurred during the pre-login handshake. (provider: TCP Provider, error: 0)\n```\n\nThere is no SQL error number anywhere in it. Our support ticket has been open for a day. In that time we have rotated the database password twice, recreated the application's database user, lengthened the connection timeout, and added and removed several connection string settings. None of it changed the message by a single character.\n\nThe team is now arguing that the message is too generic to work with and we should escalate and wait. Before we do that, tell me what this message is actually telling us and what we should check next.", + "gate": "The answer reads the message as a failure occurring in front of the login, on the network path or during protocol negotiation, and directs the investigation at the network layer and at what the two ends will negotiate, while stating plainly that no credential change and no connection string edit can affect it.", + "states": [ + "The failure occurs before any credential is exchanged, so the password, the database user and their permissions cannot be the cause.", + "The absence of a SQL error number means the connection was terminated by something between or in front of the two endpoints rather than refused by the database engine.", + "The message identifies a specific stage of the connection and is therefore diagnosable, not a generic unknown fault.", + "The change in networking is the first thing to examine, since the same code succeeds unchanged from another network." + ], + "refutes": [ + "A message of this kind with no error number carries no usable diagnostic information, so the only remaining options are to escalate, wait, or retry indefinitely.", + "This is a credential or connection string problem, so rotating the password, recreating the database user, or adjusting connection string settings is a reasonable next step.", + "Lengthening the connection timeout addresses this failure, because the connection is being cut off before it has had time to complete.", + "The database or the server is unavailable, and the right response is to wait for the service to recover." + ] + }, + "posture": ["read", "inspect"], + // Held at draft below a preview ceiling, on purpose. Re-measured 2026-09-19 at + // ten runs per arm: UNCLEAR, and nothing moved. The gate and three of the four + // statements read 10 of 10 with the skill and 10 of 10 without, the fourth 10 of + // 10 against 8 of 10, and every falsehood 0 of 10 in both arms. The model + // already answers this skill's own task correctly without it. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-to-azure-sql", + "provision-azure-sql-db", + "entra-id-auth", + "diagnose-resource-pressure", + "private-endpoints-and-firewall" + ], + "validation": { + "target": "both", + "setup": [ + "a reachable database and a client that can attempt a connection", + "an attempt from an address that no firewall rule covers", + "an attempt against a server whose public network access is disabled", + "a second attempt with a deliberately wrong password", + "a second attempt naming a database that does not exist" + ], + "assert": [ + "an address with no firewall rule returns 40615 and the skill answers with a firewall rule rather than an edit to the connection string, and accounts for the five minute propagation delay", + "a server with public network access disabled returns 47073 and is not diagnosed as a credential problem. Measured 2026-09-08 on sqllab0908614: the refusal is immediate rather than a timeout, and go-sqlcmd 1.10.0 prints the severity 10 sub-message text in full while printing no number", + "an error carrying no SQL number is classified from its message text rather than reported as unknown", + "a certificate-chain failure is answered by fixing the client trust store, never by trusting the server certificate", + "a wrong password returns 18456, and the skill names the layer that rejected it and hands it to entra-id-auth rather than answering it here or blaming the firewall", + "a database name that does not exist returns 4060, and is likewise named and handed on rather than answered as a firewall or connection-string fault", + "40613 is named as expected serverless resume and handed to connect-to-azure-sql rather than answered here", + "a refusal that arrives with no visible number is not declared unknown before -m-1 has been tried, because 47072, 47073 and 42101 are severity 10 and are converted to severity 0 before the client sees them. The TEXT still reaches the client on go-sqlcmd 1.10.0, measured 2026-09-08, so the reason is readable even where the number is not" + ], + "probes": [ + { + "sql": "SELECT message_id, severity\nFROM sys.messages\nWHERE language_id = 1033 AND message_id IN (4060, 18456)\nORDER BY message_id;", + "expect": { + "rows": [ + [4060, 11], + [18456, 14] + ] + }, + "why": "The skill's central move made executable: a number can be resolved to its severity on the engine itself rather than from memory, and the severity is what names the layer. Microsoft Learn's error number tables give 4060 severity 11, which its severity definitions define as 'the given object or entity does not exist', and 18456 severity 14, 'security'. That pair is exactly the distinction the body draws between a missing database user and a refused credential. Both numbers are core Database Engine messages present on every SQL Server engine, so this is safe on the container as well as the cloud. A red here means either the engine's message catalog moved or language_id 1033 is not installed, and the body's severity table would need re-sourcing." + }, + { + "sql": "RAISERROR (N'probe: a severity ten message does not fail the statement', 10, 1) WITH NOWAIT;", + "expect": { + "warning": 50000 + }, + "why": "Half of the reason this skill tells readers to run sqlcmd with -m-1. Microsoft Learn states that RAISERROR with a message string and no msg_id raises error number 50000, and that a severity of 10 or lower does not transfer control to a CATCH block. So this statement SUCCEEDS while emitting a numbered message, which is only visible because the runner passes -m-1. Not measured against a container by this skill's author, who was instructed not to start one; it is asserted from Learn and from the lane's own -m-1 behaviour, which eval/lib/container.mjs documents. This is the same class 47072, 47073 and 42101 belong to, and the reason a disabled public network access so often reaches a developer as an unexplained timeout." + }, + { + "sql": "RAISERROR (N'probe: a severity eleven message fails the statement', 11, 1);", + "expect": { + "error": 50000 + }, + "why": "The other half, and the pair is the point. Identical statement, severity raised by one, and now it is an error rather than a warning. That is the exact threshold sqlcmd's -b uses: it sets a non-zero exit at severity 11 and above and stays silent below it, which is why -b alone will not fail a pipeline on the severity 10 refusals above. Severity 11 is also the level 40532, 40531 and 4060 carry, so this probe fails the way the skill's own subject matter fails." + }, + { + "cmd": "sqlcmd '-?' | grep -q '^-b,--exit-on-error$' && sqlcmd '-?' | grep -q '^-m,--error-level$' && sqlcmd '-?' | grep -q '^-l,--login-timeOut$'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "Every diagnosis command in this body is sqlcmd -b -m-1 -l 30, and each of those three switches is checked to exist under that exact name in the build's own help output rather than from memory. Measured 2026-09-03 on sqlcmd 1.10.0, exit 0. Host rather than container deliberately: the reader of this skill is a developer connecting FROM their workstation into Azure SQL Database, and the client half is the half that has to be right. A red here means the body hands the reader a flag that no longer exists." + }, + { + "cmd": "az sql server firewall-rule list --help > /dev/null", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "The command the body tells a reader to run first when they see 40615, checked against the installed CLI rather than recalled. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0. --help is used deliberately so the probe proves the command surface exists without touching a subscription or creating anything. This matters here because a sibling skill's brief records az sql server update --assign-identity as a flag that does not exist on this same CLI version, so a command name from memory is not evidence." + }, + { + "cmd": "az sql server update --help | grep -q -- '--set'", + "where": "host", + "expect": { + "succeeds": true + }, + "why": "Microsoft Learn's connectivity settings page changes public network access with az sql server update --set publicNetworkAccess=\"Disabled\", a generic-update form rather than a dedicated flag, and the body reads the same property back with az sql server show --query. This checks that --set is still the shape this CLI offers. Measured 2026-09-03 on Azure CLI 2.90.0, exit 0." + } + ] + }, + "triggering": { + "explicit": [ + "Use the diagnose-connection-errors skill on this stack trace" + ], + "implicit": [ + "Cannot open server requested by the login. Client with IP address is not allowed to access the server", + "The certificate chain was issued by an authority that is not trusted", + "A connection was successfully established with the server, but then an error occurred during the pre-login handshake", + "we turned off public network access and now every connection fails with 47073, what is that", + "the gateway returned 40532 and the server name looks right to me", + "the connection just times out and there is no error number anywhere, what do I even look at" + ], + "routing": [ + "why can my database not be reached", + "the connection worked yesterday and fails today", + "my app connects fine on my laptop but gets a timeout once it is deployed" + ], + "negative": [ + "Database on server is not currently available. Please retry the connection later", + "my first connection to the serverless database fails and the second one works, what should I add", + "Login failed for user, what do I do", + "Cannot open database requested by the login. The login failed", + "write the retry policy for my data access layer", + "which driver should I use from Python", + "my query is slow, find out why", + "create a firewall rule for a new database I am provisioning" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql deleted file mode 120000 index a32141d..0000000 --- a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/ef-core-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/SKILL.md new file mode 100644 index 0000000..6752af1 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/SKILL.md @@ -0,0 +1,355 @@ +--- +name: ef-core-azure-sql +description: >- + Configures Entity Framework Core against Azure SQL Database, where retry is on by default and + redefines a transaction: the execution strategy refuses a user-initiated transaction, and the + wrapper it tells you to write replays the whole unit, so a fault after the commit writes the + row twice and reports success. Also covers UseAzureSql versus UseSqlServer, + EnableRetryOnFailure, JSON mapping and split queries. Use when a DbContext targets Azure SQL + Database, and for "EnableRetryOnFailure", "connection resiliency for EF Core", "UseAzureSql or + UseSqlServer", "AsSplitQuery", an Include returning tens of thousands of rows, "does not + support user-initiated transactions", duplicated rows after a retry, or a migration that + retypes JSON columns. Not general EF Core: pooling is connect-from-dotnet, retry + connect-to-azure-sql, identity entra-id-auth, a pipeline github-actions-for-sql. +--- + +# EF Core against Azure SQL Database + +This is the Azure SQL Database delta on top of EF Core, not an EF Core tutorial. Modelling, LINQ, +change tracking and `Include` work the same everywhere and are not repeated here. + +Command output and compiler results below were captured 2026-09-03 on +`Microsoft.EntityFrameworkCore.SqlServer` 9.0.19 with `dotnet ef` 9.0.19 on .NET SDK 8.0.421. +Runtime behaviour was measured 2026-08-27 against a live engine reporting `EngineEdition` 5. +EF Core 10 is the supported long term release and EF Core 9 is still serviced. +Open [references/retry-transactions-and-type-mapping.md](references/retry-transactions-and-type-mapping.md) when a +number below disagrees with what you are seeing, or before reproducing a run. + +## The correction + +Retry is not a setting on this database. It is on in every current answer: `EnableRetryOnFailure` +in the older corpus shape, `UseAzureSql` in the newer one. Both produce +`SqlServerRetryingExecutionStrategy`, and **turning it on silently redefines what a transaction +is.** The visible half is an exception, and it is not the problem. The problem is what an agent +writes next, because the exception dictates its own remedy and the remedy is incomplete: + +```csharp +// Retry on, so this unit is replayed on a transient fault. Measured: a fault arriving after +// Commit runs the delegate again, inserts the order a SECOND time, and Execute returns +// success. Nothing is thrown. Nothing is logged. The caller is told it worked. +var strategy = context.Database.CreateExecutionStrategy(); +strategy.Execute(() => +{ + using var tx = context.Database.BeginTransaction(); + context.Orders.Add(order); + context.SaveChanges(); + tx.Commit(); +}); +``` + +**A retriable unit is a unit that will be run more than once.** Retry does not make a transaction +resilient. It makes it repeatable, and a repeatable write that is not idempotent is a duplicate +write waiting for a network fault it was configured to expect. Both call shapes carry this +identically, so switching call does not fix it and staying on the old call does not cause it. + +## What actually refuses, and what does not + +Measured on 9.0.19 against the live engine. The commonly repeated claim, that +`Database.BeginTransaction` throws, is wrong. + +| Call, with the retrying strategy configured | Result | +|---|---| +| `Database.BeginTransaction()` on its own | **Succeeds.** No exception | +| A LINQ query while that transaction is open | Throws `InvalidOperationException` in about 60 ms | +| `SaveChanges` while that transaction is open | Throws `InvalidOperationException` in about 20 ms | +| `Database.ExecuteSqlRaw` while that transaction is open | **Succeeds.** Not guarded at all | +| `SaveChanges` with no explicit transaction | Succeeds. EF's own transaction is a unit it created | + +The guard fires on EF's own pipeline, not on raw SQL, and on the first execution of the path rather +than under load: + +> `InvalidOperationException: The configured execution strategy 'SqlServerRetryingExecutionStrategy' +> does not support user-initiated transactions. Use the execution strategy returned by +> 'DbContext.Database.CreateExecutionStrategy()' to execute all the operations in the transaction as +> a retriable unit.` + +**This exception is cheap**: deterministic, milliseconds, on the first run of the path, and it names +its own fix. Treat it as a compiler error that arrives late. **The fix it names is not the whole +fix**, and nothing after it fails loudly. That is the next section, and it is why this skill exists. + +## The measured duplicate write + +Same delegate, same configured strategy, one variable: where the transient fault lands. + +| Fault arrives | Delegate runs | Rows committed | What the caller sees | +|---|---|---|---| +| Before `Commit` | 2 | **1** | Success. Correct, the first attempt rolled back | +| After `Commit` | 2 | **2** | Success. **No exception, no warning, no log line** | + +The window is the commit acknowledgement. The commit reached the server, the acknowledgement did +not reach the client, and the strategy cannot tell that apart from a commit that never happened, so +it replays a unit that already succeeded. Learn calls this the idempotency issue and says it can +lead to data corruption. The blast radius is a silent double write on a transactional path, in +production only, on exactly the transient faults this database is configured to expect. + +### Making the unit safe to run twice + +In order of preference. + +1. **Give the write a natural idempotency key and a unique index**, so a replay collides with + `Msg 2601` instead of duplicating. This is the only defence that survives a process crash + between the two attempts. +2. **Hand the transaction to `ExecuteInTransaction` with a `verifySucceeded` predicate.** The + strategy owns the transaction and, when the commit fails with an unknown outcome, asks the + predicate whether the work already landed before retrying: + +```csharp +var strategy = db.Database.CreateExecutionStrategy(); +db.Orders.Add(order); + +await strategy.ExecuteInTransactionAsync( + db, + operation: (context, cancellationToken) => + context.SaveChangesAsync(acceptAllChangesOnSuccess: false, cancellationToken), + verifySucceeded: (context, cancellationToken) => + context.Orders.AsNoTracking().AnyAsync(o => o.Key == order.Key, cancellationToken), + cancellationToken: ct); + +db.ChangeTracker.AcceptAllChanges(); +``` + +`acceptAllChangesOnSuccess: false` is load bearing, not decoration: it leaves the entity `Added` so +the same unit can be replayed if the commit is what failed, and `AcceptAllChanges` afterwards is +what finishes the save. The verifying context needs its own execution strategy, because the +connection that just failed is likely to fail again during the check. + +3. **Only then** the bare `CreateExecutionStrategy().ExecuteAsync` wrapper, and only for units that + are already safe to run twice. The delegate must not capture entities tracked by an outer + context, must not rely on store generated keys, and must not send an email, enqueue a message or + charge a card inside it. + +Some units cannot be made idempotent, and for those the choice is to accept the duplicate risk or +to narrow the retry set, never to disable retry so the exception goes away. + +## What retry costs when it does not help + +`UseAzureSql` defaults to 6 retries with a 30 second cap, so 7 attempts, and measured end to end a +permanently failing operation gives up after **57 seconds**, with attempts starting at 0.0, 0.0, +1.0, 4.1, 11.6, 26.9 and 56.9 seconds. A request thread is held for that whole minute, and behind a +30 second gateway timeout the caller has already given up. Size the retry budget against the +deadline of the thing calling you, and put a `CancellationToken` through every async call. + +## UseAzureSql, and the near miss + +From **EF Core 9** there is a call named for this database, and Learn's own Azure SQL Database tab +uses it. It does **not** delete `EnableRetryOnFailure`; it changes the default from off to on: + +```csharp +options.UseAzureSql(connectionString, o => o.UseCompatibilityLevel(170)); +``` + +| | `UseSqlServer` | `UseAzureSql` | +|---|---|---| +| Engine type | `SqlServer` | `AzureSql` | +| Execution strategy | `SqlServerExecutionStrategy`, no retry | `SqlServerRetryingExecutionStrategy`, 6 retries, 30 second cap | +| Default compatibility level, EF Core 9 | 150 | 150 | +| Default compatibility level, EF Core 10 | 150 | **170** | +| Native `json` column type, EF Core 10 | only at level 170 or above | on by default | + +- **It is a compile time API.** On EF Core 8 the call does not exist and the build says + `error CS1061: 'DbContextOptionsBuilder' does not contain a definition for 'UseAzureSql'`. + Upgrade the provider package rather than inventing an extension method. +- **When `UseSqlServer` is called by code you cannot change**, EF Core 9 has a first party answer + that is not obsolete, and it compiled clean here: `options.ConfigureSqlEngine(c => + c.EnableRetryOnFailureByDefault());`. `UseSqlServer(cs, o => o.UseAzureSqlDefaults(true))` also + installs the retrying strategy, but it warns `CS0618: 'UseAzureSqlDefaults(bool)' is obsolete` and + leaves the engine type as `SqlServer`, so the type mapping does not follow. +- **Retry numbers are defaults, not overrides.** Supplying them alongside `UseAzureSql` replaces the + 6 and the 30 seconds. Do that with a measured reason, never by copying a blog post: + +```csharp +options.UseAzureSql( + connectionString, + o => o.EnableRetryOnFailure(maxRetryCount: 3, maxRetryDelay: TimeSpan.FromSeconds(10), errorNumbersToAdd: null)); +``` + +Whether to retry at all is not a question on this database. **The split is clean: +`connect-to-azure-sql` owns retry at the driver and connection level, this skill owns the EF Core +execution strategy above it and the transaction semantics that only exist up here.** + +## Compatibility level, and the migration nobody expects + +On EF Core 10, `UseAzureSql` defaults the compatibility level to **170**, and at 170 the provider +maps JSON to the native `json` column type instead of `nvarchar(max)`. Learn states it as a +breaking change: upgrading an application that already uses `UseAzureSql` generates a migration that +alters every existing `nvarchar(max)` JSON column to `json`: data movement on a live table, +produced by a change that reads as configuration. + +Generate the migration and read it before it reaches a database: + +```bash +dotnet ef migrations add UpgradeToEfCore10 +dotnet ef migrations script --idempotent --output migrate.sql +grep -nE 'ALTER COLUMN|json' migrate.sql +``` + +Then choose, rather than inherit: + +- **State the level.** Learn names a level below 170 as the opt out, and its own example is + `o.UseCompatibilityLevel(160)`, which keeps `nvarchar(max)`. +- **Pin one column** with `HasColumnType("nvarchar(max)")` where only part of the model must wait. +- **Take the retype deliberately**, in a window sized for the table, because the `ALTER` rewrites it. + +The level does **not** gate `vector(n)`, which EF Core 10 surfaces as `SqlVector` and +`EF.Functions.VectorDistance`, so do not raise it expecting to unlock vectors. Storing embeddings is +`vector-search-azure-sql`. + +## Type mapping, honestly scoped + +Most EF Core type mapping advice applies to any engine this provider talks to and belongs to +`design-azure-sql-schema`. Two things are different here. + +**`DateTime` with `HasDefaultValueSql("GETDATE()")`.** Azure SQL Database offers no time zone +choice: `CURRENT_TIMEZONE()` returns UTC and `GETDATE()` equals `SYSUTCDATETIME()`, in the container +and in the cloud alike. A model carried over from an engine running on local time changes meaning +without changing shape. Prefer `DateTimeOffset`, or `SYSUTCDATETIME()`, and be explicit either way. + +**An indexed `string` with no `MaxLength` is narrowed to `nvarchar(450)`**, and neither the index +nor the data fails silently. From the script generated above, for one indexed string property: + +```text +[Name] nvarchar(450) NOT NULL, +CREATE INDEX [IX_Orders_Name] ON [Orders] ([Name]); +``` + +The `CREATE INDEX` succeeds and the engine then refuses anything longer, with +`Msg 2628, String or binary data would be truncated in table '
', column ''`. Adding +the index later to a column already holding longer values fails the same way on the `ALTER COLUMN`, +so the migration stops rather than trimming data. Set `HasMaxLength` deliberately instead of +discovering the 450 at deployment. + +## Single and split queries over a wide area network + +`Include` of two sibling collections produces a cross product. That is engine independent. **What +changes here is the arithmetic of the fix**, because each extra query is a real network round trip. + +- **Prefer split** when the cartesian product is large: moving duplicated megabytes costs more. +- **Prefer single** for small collections and chatty request paths. +- **Measure.** The break even point moves with row width and with distance. +- Split queries are not one consistent read unless wrapped in a snapshot or serializable + transaction, which puts them back under everything above. + +Slow query triage is `diagnose-slow-query`. + +## Migrations against a cloud database + +Doctrine is `schema-migrations-safely`, the declarative alternative `sql-database-projects`, the +deployment step `ef-core-migrations-azure-sql`. Only the Azure SQL Database parts are here: + +```bash +dotnet ef database update \ + --connection "Server=tcp:your-server.database.windows.net,1433;Database=your-database;Authentication=Active Directory Default;Encrypt=Mandatory;" +``` + +- **It runs from wherever it is invoked**, so that machine needs a server firewall rule for its own + address and a principal that can change the schema. Neither is the application's runtime identity, + and `--connection` is how the two are kept apart: it overrides the string in `AddDbContext` or + `OnConfiguring` without editing the application. +- **The database itself is not created by a migration.** That is `provision-azure-sql-db`. +- **Do not migrate from application startup here.** EF Core 9 added a database wide lock, so the + concurrency objection an agent will recall is out of date. The objection that survives is + permissions: startup migration means the runtime identity holds schema rights permanently. + +## Connecting with an identity rather than a password + +Nothing in the `DbContext` changes. `UseAzureSql` takes the same connection string a bare driver +would, so passwordless access stays a connection string concern: + +```text +Server=tcp:your-server.database.windows.net,1433;Database=your-database;Authentication=Active Directory Default;Encrypt=Mandatory; +``` + +Getting an identity to a working connection is `entra-id-auth`. Packages, authentication modes and +pooling are `connect-from-dotnet`. + +## Check it worked + +Three checks, in the order they catch things. + +**The context is on the Azure SQL Database path at all.** Nothing in the connection string says so, +and this is the only command that prints it: + +```bash +dotnet ef dbcontext info +``` + +Expect `Options: EngineType=AzureSql`. Under `UseSqlServer`, with or without a hand written +`EnableRetryOnFailure`, the same line reads `Options: EngineType=SqlServer`, which is how a +`UseAzureSqlDefaults` configuration is caught: it retries, and it still prints `SqlServer`. + +**The model and the migrations agree**, before a deployment discovers they do not: + +```bash +dotnet ef migrations has-pending-model-changes +``` + +Expect exit code 0 and `No changes have been made to the model since the last migration.` Exit code +1 and `Changes have been made to the model since the last migration.` is the failure, and it is the +one that becomes an unexplained `ALTER` in somebody else's release. + +**A replayed unit collides instead of duplicating.** An idempotency key is only a defence if the +index over it is unique: + +```sql +SELECT i.name, i.is_unique +FROM sys.indexes AS i +WHERE i.object_id = OBJECT_ID('dbo.Orders') AND i.is_unique = 1; +``` + +Expect one row, the index over the key column. With `is_unique` 1 a replayed insert fails with +`Msg 2601, Cannot insert duplicate key row in object 'dbo.Orders' with unique index`; with a +non-unique index the replay writes a second row and the whole defence is decorative. + +## Do not + +- Do not treat the execution strategy exception as the end of the work. It is the visible half, and + the silent half is a duplicate write. +- Do not put a hand written `BeginTransaction` and `Commit` inside a retried delegate without an + idempotency key or a `verifySucceeded` predicate. It compiles, it passes review, and it doubles a + row on the fault it was configured to expect. +- Do not drop `acceptAllChangesOnSuccess: false` from the `ExecuteInTransaction` form. Without it + the entity is marked `Unchanged` and the replay saves nothing. +- Do not assume `Database.BeginTransaction` throws. It succeeds, and the next query or `SaveChanges` + throws. Raw SQL inside that transaction is not guarded at all. +- Do not disable retry to make a transaction error disappear, and do not leave the retry budget + larger than the caller's timeout. +- Do not claim `UseAzureSql` removes `EnableRetryOnFailure`. It sets its default. +- Do not reach for `UseAzureSqlDefaults(true)`. It is obsolete, and it leaves the engine type + unchanged; `ConfigureSqlEngine(c => c.EnableRetryOnFailureByDefault())` is the supported form. +- Do not upgrade to EF Core 10 and switch to `UseAzureSql` in the same change without reading the + next migration. +- Do not add `AsSplitQuery` everywhere by reflex. Over a network it can cost more than it saves. +- Do not apply migrations from application startup against a cloud database. +- Do not leave an indexed string property without `HasMaxLength`, and do not leave two sibling + collection `Include` calls without `AsSplitQuery` or `AsSingleQuery`. +- Do not give the application's runtime identity schema permission, and do not put a connection + string, password or server hostname in source. +- Do not use this skill for a different object relational mapper, or for raw + `Microsoft.Data.SqlClient` work. + +## References + +- [Every run behind the tables above, and how to reproduce one](references/retry-transactions-and-type-mapping.md) +- [Connection resiliency](https://learn.microsoft.com/ef/core/miscellaneous/connection-resiliency): + execution strategies, the transaction rule, and the four documented answers to a commit that fails + with an unknown outcome. Read it before writing anything transactional. +- [Microsoft SQL Server database provider](https://learn.microsoft.com/ef/core/providers/sql-server/): + the first party statement of `UseAzureSql`, `UseCompatibilityLevel` and `ConfigureSqlEngine`. +- [Breaking changes in EF Core 10](https://learn.microsoft.com/ef/core/what-is-new/ef-core-10.0/breaking-changes): + read before an EF Core 10 upgrade, for the JSON retype and the compatibility level that gates it. +- [Single vs split queries](https://learn.microsoft.com/ef/core/querying/single-split-queries): + cartesian explosion and every characteristic of split queries. +- Underneath this skill: `connect-to-azure-sql` for retry and pool sizing at the connection level, + `connect-from-dotnet` for the driver and packages, `entra-id-auth` for a passwordless identity. +- Beside it: `design-azure-sql-schema` for string sizing and collation, `schema-migrations-safely` + and `ef-core-migrations-azure-sql` for how migrations reach a database. diff --git a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/references/retry-transactions-and-type-mapping.md b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/references/retry-transactions-and-type-mapping.md new file mode 100644 index 0000000..67e8470 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/references/retry-transactions-and-type-mapping.md @@ -0,0 +1,378 @@ +# Retry, transactions and type mapping: what was measured, and how to measure it again + +## Contents + +- [Why this file exists](#why-this-file-exists) +- [Versions in play on the verification date](#versions-in-play-on-the-verification-date) +- [Reproducing the measurements](#reproducing-the-measurements) +- [Measurement 1: the duplicate write, which is the headline](#measurement-1-the-duplicate-write-which-is-the-headline) +- [Measurement 2: what the execution strategy actually refuses](#measurement-2-what-the-execution-strategy-actually-refuses) +- [Measurement 3: what retry costs when it does not help](#measurement-3-what-retry-costs-when-it-does-not-help) +- [Measurement 4: UseAzureSql exists only from EF Core 9](#measurement-4-useazuresql-exists-only-from-ef-core-9) +- [Measurement 5: the four ways to reach a retrying strategy](#measurement-5-the-four-ways-to-reach-a-retrying-strategy) +- [Measurement 6: compatibility level defaults](#measurement-6-compatibility-level-defaults) +- [Measurement 7: what an unconfigured model maps to, and what the engine then does](#measurement-7-what-an-unconfigured-model-maps-to-and-what-the-engine-then-does) +- [Measurement 8: the server clock](#measurement-8-the-server-clock) +- [Measurement 9: the dotnet ef tool runs, 2026-09-03](#measurement-9-the-dotnet-ef-tool-runs-2026-09-03) +- [What was not verified by execution](#what-was-not-verified-by-execution) +- [Claims this file previously got wrong](#claims-this-file-previously-got-wrong) + +## Why this file exists + +Every table in `SKILL.md` came from a compiler or a running process rather than from a +documentation page, because the questions are about API surface, defaults and runtime behaviour, +which go stale between a release and the article describing it. This file records what was run so +the claims can be rechecked against a version that did not exist on the verification date. + +Runtime behaviour: **2026-08-27**, against a live engine. Tool and compiler output: **2026-09-03**, +in measurement 9. + +## Versions in play on the verification date + +| Package or product | Version | +|---|---| +| `Microsoft.EntityFrameworkCore.SqlServer`, latest supported long term release | 10.0.11 | +| `Microsoft.EntityFrameworkCore.SqlServer`, latest EF Core 9 patch | 9.0.19 | +| `Microsoft.EntityFrameworkCore.SqlServer`, next release | 11.0.0 preview | +| Versions compiled | 8.0.11 and 9.0.19 | +| Version executed against an engine | 9.0.19 | +| Version read as assembly metadata only | 10.0.11 | +| Engine the runtime measurements ran against | Azure SQL Database container, `EngineEdition` 5, `Edition` `SQL Azure`, `ProductVersion` 12.0.2000.8 | + +EF Core 10 requires the .NET 10 runtime, so on a host with an earlier software development kit it +can be inspected but not executed. Its public surface was read with a metadata load context, which +needs no matching runtime. + +## Reproducing the measurements + +```bash +dotnet new console -o efprobe +cd efprobe +dotnet add package Microsoft.EntityFrameworkCore.SqlServer --version 9.0.19 +``` + +The API level measurements need no database: `CreateExecutionStrategy` and the model builder never +open a connection. Measurements 1, 2, 3, 7 and 8 need a reachable Azure SQL engine, which is what +the local container is for. + +A transient fault cannot be summoned on demand, so it is **simulated by declaring an error number +transient**. `EnableRetryOnFailure` takes a list of additional error numbers, and `RAISERROR` with a +message string raises 50000, so: + +```csharp +options.UseAzureSql(connectionString, o => o.EnableRetryOnFailure(3, TimeSpan.FromSeconds(1), new[] { 50000 })); + +using var context = new AppDbContext(options.Options); +context.Database.ExecuteSqlRaw("RAISERROR('dropped', 16, 1)"); +``` + +makes the strategy treat that statement as a transient failure and retry the unit. The retry +machinery under test is the real one; only the classification of the error is arranged. + +## Measurement 1: the duplicate write, which is the headline + +One delegate, handed to `CreateExecutionStrategy().Execute`, containing a hand written +`BeginTransaction`, a `SaveChanges` and a `Commit`. The only variable is where the simulated +transient fault lands. + +| Fault arrives | Delegate runs | Rows committed | What `Execute` returned | +|---|---|---|---| +| Before `Commit` | 2 | 1 | success | +| After `Commit` | 2 | **2** | **success** | + +Captured output: + +```text +fault BEFORE commit: Execute returned SUCCESS, delegate ran 2x, rows = 1 +fault AFTER commit: Execute returned SUCCESS, delegate ran 2x, rows = 2 +``` + +The same shape under the corpus configuration, `UseSqlServer` with `EnableRetryOnFailure`, is +identical: + +```text +Execute returned SUCCESS +delegate ran 2x, rows = 2 +``` + +So the hazard is a property of retry, not of `UseAzureSql`. The window is the commit +acknowledgement: the commit reached the server, the acknowledgement did not reach the client, and +the strategy cannot distinguish that from a commit that never happened. + +The guarded form, `ExecuteInTransaction` with a `verifySucceeded` predicate, was run against the +same fault and committed exactly one row: + +```text +ExecuteInTransaction returned after 2 delegate runs +rows committed = 1 +``` + +## Measurement 2: what the execution strategy actually refuses + +On 9.0.19 against the live engine, with the retrying strategy configured: + +```text +BeginTransaction alone OK (11 ms) +BeginTransaction then a LINQ query THREW InvalidOperationException (63 ms) +BeginTransaction then SaveChanges THREW InvalidOperationException (20 ms) +BeginTransaction then ExecuteSqlRaw only OK (25 ms) +no transaction, SaveChanges OK (38 ms) +``` + +Three things this settles: + +- **`Database.BeginTransaction` does not throw.** The common telling of this trap, repeated by the + previous revision of `SKILL.md`, puts the exception on that call. It is on the first EF operation + performed while the transaction is open. +- **Raw SQL is not guarded.** `ExecuteSqlRaw` inside a user-initiated transaction is permitted. +- **The refusal is immediate and deterministic**, tens of milliseconds on the first execution of the + path, so it is not a load-dependent failure and any run of the path finds it. + +The message, captured rather than quoted: + +```text +The configured execution strategy 'SqlServerRetryingExecutionStrategy' does not support +user-initiated transactions. Use the execution strategy returned by +'DbContext.Database.CreateExecutionStrategy()' to execute all the operations in the transaction as +a retriable unit. +``` + +It names a remedy and says nothing about idempotency, which is measurement 1. + +## Measurement 3: what retry costs when it does not help + +At the `UseAzureSql` defaults, 6 retries with a 30 second cap, against an operation that fails every +time: + +```text +RetryLimitExceededException after 7 attempts, 56.9 s total +attempt start times (s): 0.0, 0.0, 1.0, 4.1, 11.6, 26.9, 56.9 +``` + +Seven attempts, so the delays are roughly 0, 1, 3, 7.5, 15 and 30 seconds, the last one at the cap. +One failing operation holds a thread for the better part of a minute. + +## Measurement 4: UseAzureSql exists only from EF Core 9 + +Against 8.0.11, `optionsBuilder.UseAzureSql(...)` does not compile: + +```text +error CS1061: 'DbContextOptionsBuilder' does not contain a definition for 'UseAzureSql' +and no accessible extension method 'UseAzureSql' accepting a first argument of type +'DbContextOptionsBuilder' could be found +``` + +Against 9.0.0, 9.0.19 and 10.0.11 the method is present on +`SqlServerDbContextOptionsExtensions`, in eight overloads that mirror the `UseSqlServer` set exactly. +The options builder handed to the callback is `AzureSqlDbContextOptionsBuilder`. It carries +`EnableRetryOnFailure` in four overloads and `UseCompatibilityLevel(int)`. `UseAzureSql` therefore +**defaults** retry rather than removing the knob. + +## Measurement 5: the four ways to reach a retrying strategy + +All measured on 9.0.19. The engine type column is the one that gets overlooked. + +| Configuration | Engine type | Strategy | Retry count and cap | +|---|---|---|---| +| `UseAzureSql(cs)` | `AzureSql` | retrying | 6, 30 seconds | +| `UseAzureSql(cs, o => o.EnableRetryOnFailure(3, TimeSpan.FromSeconds(10), null))` | `AzureSql` | retrying | 3, 10 seconds | +| `UseSqlServer(cs, o => o.UseAzureSqlDefaults(true))` | **`SqlServer`** | retrying | 6, 30 seconds | +| `UseSqlServer(cs, o => o.EnableRetryOnFailure(6, TimeSpan.FromSeconds(30), null))` | **`SqlServer`** | retrying | 6, 30 seconds | + +Captured: + +```text +UseSqlServer engine=SqlServer strategy=SqlServerExecutionStrategy retries=False +UseAzureSql engine=AzureSql strategy=SqlServerRetryingExecutionStrategy retries=True + MaxRetryCount=6 MaxRetryDelay=00:00:30 +UseSqlServer+UseAzureSqlDefaults engine=SqlServer strategy=SqlServerRetryingExecutionStrategy retries=True + MaxRetryCount=6 MaxRetryDelay=00:00:30 +``` + +## Measurement 6: compatibility level defaults + +Read from the provider's own options on 9.0.19, and from the 10.0 source constants: + +```text +UseSqlServer: EngineType=SqlServer SqlServerCompatibilityLevel=150 AzureSqlCompatibilityLevel=150 +UseAzureSql: EngineType=AzureSql SqlServerCompatibilityLevel=150 AzureSqlCompatibilityLevel=150 +``` + +For 10.0.11 the defaults are static readonly fields rather than literals, so they were decoded from +the type initializer of `SqlServerOptionsExtension` in the shipped assembly: + +```text +SqlServerDefaultCompatibilityLevel = 150 +AzureSqlDefaultCompatibilityLevel = 170 +AzureSynapseDefaultCompatibilityLevel = 30 +``` + +| Provider version | `UseSqlServer` default | `UseAzureSql` default | +|---|---|---| +| 9.0.19 | 150 | 150 | +| 10.0.11 | 150 | **170** | + +Level 170 is the gate on the native `json` mapping, and on that alone. **It does not gate +`SqlVector`.** That last point is read from the provider's type mapping source, where the +type resolves with no compatibility check of any kind, and not measured here: it depends on EF Core +10 and on the server supporting the `vector` type, and raising the level does not unlock it. + +## Measurement 7: what an unconfigured model maps to, and what the engine then does + +An entity with no explicit configuration and one index on a string property, on 9.0.19: + +```sql +CREATE TABLE [Blogs] ( + [Id] int NOT NULL IDENTITY, + [Name] nvarchar(450) NOT NULL, + [Price] decimal(18,2) NOT NULL, + [CreatedAt] datetime2 NOT NULL, + [Contact] nvarchar(max) NOT NULL, + CONSTRAINT [PK_Blogs] PRIMARY KEY ([Id]) +); +CREATE INDEX [IX_Blogs_Name] ON [Blogs] ([Name]); +``` + +`Name` is `nvarchar(450)`, not `nvarchar(max)`: indexing a string property narrows it. The question +the previous revision answered wrongly is what happens next. Inserting 500 characters through +`SaveChanges`: + +```text +actual column type: Name nvarchar(450) +THREW DbUpdateException: Msg 2628 / String or binary data would be truncated in table +'appdb.dbo.Blogs', column 'Name'. +``` + +And adding the index later, over a column that already holds 600 characters: + +```text +ALTER THREW Msg 2628: String or binary data would be truncated in table 'appdb.dbo.Wide', +column 'Name'. The statement has been terminated. +``` + +So **neither the write nor the migration is silent**. The `CREATE INDEX` succeeds and the engine +refuses the data, at insert time and at `ALTER COLUMN` time alike. The correct advice is to set +`HasMaxLength` deliberately, not to fear silent data loss. + +`Price` is `decimal(18,2)`, and the provider carries a warning whose exact text is: "No store type +was specified for the decimal property '{property}' on entity type '{entityType}'. This will cause +values to be silently truncated if they do not fit in the default precision and scale." Provider +behaviour rather than Azure SQL Database behaviour. + +## Measurement 8: the server clock + +On the container, from the same session as the rest: + +```text +GETDATE() = Aug 28 2026 5:36AM +SYSUTCDATETIME() = 2026-08-28 05:36:37.3657578 +SYSDATETIMEOFFSET() = 2026-08-28 05:36:37.3746771 +00:00 +CURRENT_TIMEZONE() = (UTC) Coordinated Universal Time +host local now = 8/27/2026 11:36:37 PM +``` + +The host was six hours behind, and `GETDATE()` returned the UTC time, so the container agrees with +the cloud rather than with the machine running it. A `GETDATE()` default therefore behaves the same +locally and in Azure, and only a model carried over from an engine running on local time changes +meaning. + +## Measurement 9: the dotnet ef tool runs, 2026-09-03 + +Host: .NET SDK 8.0.421, `dotnet ef` 9.0.19 installed with +`dotnet tool install --global dotnet-ef --version 9.0.19`, provider package 9.0.19. EF Core 9 +targets `net8.0`, so `UseAzureSql` is reachable from an SDK 8 host. + +The fixture: a console project with one entity carrying a non-unique index on a `string` property, +a unique index on a `Guid` key, and a `string[]` collection property. + +**`dotnet ef dbcontext info` is the only command that prints the engine type.** Under +`options.UseAzureSql(...)`: + +```text +Type: AppDbContext +Provider name: Microsoft.EntityFrameworkCore.SqlServer +Database name: appdb +Data source: tcp:127.0.0.1,1433 +Options: EngineType=AzureSql +``` + +Changing that one call to `options.UseSqlServer(...)`, with nothing else touched, and rerunning: + +```text +Options: EngineType=SqlServer +``` + +The provider name is identical in both, and so is the connection string. That line is the only +thing that tells the two configurations apart. + +**`dotnet ef migrations has-pending-model-changes` carries its answer in the exit code.** + +```text +$ dotnet ef migrations has-pending-model-changes ; echo $? +No changes have been made to the model since the last migration. +0 +$ dotnet ef migrations has-pending-model-changes ; echo $? +Changes have been made to the model since the last migration. Add a new migration. +1 +``` + +The second run followed adding one property to the entity and nothing else. Exit 1 is usable in CI. + +**The generated script, from `dotnet ef migrations script --idempotent --output migrate.sql`**, on +EF Core 9: + +```text +[Name] nvarchar(450) NOT NULL, +[Tags] nvarchar(max) NOT NULL, +CREATE UNIQUE INDEX [IX_Orders_IdempotencyKey] ON [Orders] ([IdempotencyKey]); +CREATE INDEX [IX_Orders_Name] ON [Orders] ([Name]); +``` + +`Name` is the indexed string, narrowed to 450, confirming measurement 7 on a second host. `Tags` is +the `string[]`, `nvarchar(max)` on EF Core 9, and it is exactly the column EF Core 10 with +`UseAzureSql` retypes to `json`. + +**Compiler results on 9.0.19**, three configurations built in one project: + +```text +warning CS0618: 'SqlServerDbContextOptionsBuilder.UseAzureSqlDefaults(bool)' is obsolete: +'Use UseAzureSql instead of UseSqlServer with UseAzureSqlDefaults.' +``` + +That is the only diagnostic in the build. `ConfigureSqlEngine(c => c.EnableRetryOnFailureByDefault())`, +which Microsoft Learn gives as the EF Core 9 answer for a `UseSqlServer` call you do not control, +compiled with no warning, and so did `UseAzureSql(cs, o => o.UseCompatibilityLevel(170))` and the +`ExecuteInTransactionAsync(db, operation:, verifySucceeded:, cancellationToken:)` form quoted in +`SKILL.md`. + +## What was not verified by execution + +Stated so that nobody reads this file as more than it is. + +- **EF Core 10 was read, not run.** Its public surface and its compatibility level constants are + first hand, decoded from the 10.0.11 assembly on this host; its runtime behaviour is taken from + the release notes. +- **The `SqlVector` gating claim is a source reading**, carried forward from the previous + revision, not something this host executed. +- **Measurement 9 opened no connection.** `dbcontext info`, `migrations script` and + `has-pending-model-changes` are design-time commands; the server on the connection string was not + running. +- **The JSON retype on first migration** is quoted from the EF Core 10 release notes, which state + that existing `nvarchar` JSON columns are changed to `json` by the first migration, and name the + two ways to opt out. It was not observed against a database. +- **The transient fault was simulated**, by declaring error 50000 retriable, rather than produced by + a real network partition. The retry machinery, the delegate replay and the row counts are real. +- **The single and split query trade-off was not benchmarked.** It is a latency argument, and this + measurement host is a container on the same machine, which is the one topology where the argument + does not apply. + +## Claims this file previously got wrong + +Kept rather than deleted, because a reference file that quietly rewrites itself teaches nothing. + +- **"No live database was involved."** True of the previous revision, and it is what allowed the two + errors below. An engine that would not start was accepted as a reason to quote documentation. +- **"Indexing a string property narrows it silently, and the silent narrowing is what truncates + data."** Wrong. The narrowing is real, the truncation is not silent: the engine raises Msg 2628. +- **The exception was placed on `Database.BeginTransaction`.** Wrong. That call succeeds, and the + next EF operation inside the transaction is what throws. diff --git a/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..b3be864 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/ef-core-azure-sql/skill.spec.jsonc @@ -0,0 +1,143 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "ef-core-azure-sql", + "domain": "schema-and-orm", + "value": ["not-in-training-data", "corrects-a-wrong-assumption"], + "correction": "Every current answer for Entity Framework Core on Azure SQL Database turns retry on, whether by the corpus shape UseSqlServer with EnableRetryOnFailure or by the EF Core 9 call UseAzureSql, and both install SqlServerRetryingExecutionStrategy. The agent then treats the InvalidOperationException that follows, 'does not support user-initiated transactions', as the whole problem, because the exception dictates its own remedy: wrap the transaction in a delegate passed to Database.CreateExecutionStrategy(). That remedy is incomplete and its incompleteness is silent. Measured on 2026-08-27 against a live Azure SQL engine on 9.0.19, a transient fault arriving after the Commit inside that delegate runs the delegate a second time, commits the row twice, and returns success to the caller with no exception, no warning and no log line, identically under UseSqlServer and under UseAzureSql. Turning retry on does not make a transaction resilient, it makes it repeatable, so a retriable unit is correct only once it carries an idempotency key or is handed to ExecuteInTransaction with a verifySucceeded predicate. Two commonly repeated claims are also wrong and were measured wrong: Database.BeginTransaction itself succeeds and the next query or SaveChanges is what throws, and ExecuteSqlRaw inside that transaction is not guarded at all.", + "value_probe": { + "task": "Our ASP.NET Core service uses Entity Framework Core 9 against a database in Azure. We turned on the built in retrying behaviour for transient faults when we moved to Azure.\n\nA checkout path writes an order and then writes a ledger entry, and we needed both or neither, so we wrapped the two SaveChanges calls in a transaction. That immediately threw at runtime telling us the configured execution strategy does not support user initiated transactions. We followed what the exception told us to do and wrapped the whole thing in a delegate handed to the execution strategy, roughly:\n\n```csharp\nvar strategy = db.Database.CreateExecutionStrategy();\nawait strategy.ExecuteAsync(async () =>\n{\n await using var tx = await db.Database.BeginTransactionAsync();\n db.Orders.Add(order);\n await db.SaveChangesAsync();\n db.Ledger.Add(entry);\n await db.SaveChangesAsync();\n await tx.CommitAsync();\n});\n```\n\nThe exception went away and this has been in production for three months. Roughly once a fortnight, always during a period when the database is visibly having a bad few minutes, a customer is charged twice and we find two identical orders seconds apart. There is no exception in our logs, no warning, and no failed request. The customer's own request returned a success.\n\nIs this code correct, and if not, what do we do about it?", + "gate": "The answer identifies that the retried unit can be run a second time after its first run already committed, so the write is repeated and reported as success, and requires the unit to be made safe to repeat, whether by an idempotency key, a uniqueness constraint the second attempt violates, or a check that confirms whether the first attempt succeeded before repeating it.", + "states": [ + "Wrapping the transaction in the execution strategy delegate makes the unit repeatable, and does not by itself make it safe to repeat.", + "A transient fault arriving after the commit causes the entire delegate to run again, producing a second committed write.", + "The duplicate is reported to the caller as a success, with no exception, warning or log entry, which is why nothing appears in the logs.", + "Correctness requires either an identifier that makes a repeat a no operation, or a check that establishes whether the first attempt committed before it is retried." + ], + "refutes": [ + "Wrapping the transaction in a delegate passed to the execution strategy makes the transaction safe to retry, so the duplicates must originate somewhere else, such as the client, a load balancer, or a message being delivered twice.", + "A retry can only re-run a transaction that failed to commit, so a committed transaction is never repeated by the retrying behaviour.", + "Turning retry on is what makes a write resilient against transient faults on this service.", + "The fix is to raise the retry count, lengthen the delay between attempts, or turn the retrying behaviour off for this code path." + ] + }, + "posture": ["read", "write", "execute"], + // Held at draft below a preview ceiling, on purpose. The value measurement of + // 2026-09-05, ten runs per arm, returned NEITHER: the gate and two statements at + // 10 of 10 in both arms, the others within one run of level, and every falsehood + // at 0 or 1 of 10 in both. The model already answers this skill's own task + // correctly without it. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-from-dotnet", + "connect-to-azure-sql", + "entra-id-auth", + "design-azure-sql-schema", + "schema-migrations-safely", + "sql-database-projects", + "ef-core-migrations-azure-sql", + "dapper-and-ado-net-azure-sql", + "build-app-on-azure-sql" + ], + "validation": { + "target": "both", + "setup": [ + "a .NET project referencing Microsoft.EntityFrameworkCore.SqlServer 9.0 or later", + "an Azure SQL database reachable from the build host, or the local container" + ], + "assert": [ + "every delegate passed to Database.CreateExecutionStrategy carries an idempotency key with a unique index, or uses ExecuteInTransaction with a verifySucceeded predicate, or records why replaying it cannot duplicate anything", + "no non idempotent side effect, such as sending mail, enqueuing a message or taking a payment, sits inside a retried delegate", + "no code path calls Database.BeginTransaction outside a delegate passed to Database.CreateExecutionStrategy", + "the retry budget is smaller than the deadline of whatever calls the application, and a CancellationToken reaches every async database call", + "dotnet ef dbcontext info reports Options: EngineType=AzureSql, rather than EngineType=SqlServer with a hand written EnableRetryOnFailure", + "indexed string properties state HasMaxLength rather than inheriting the nvarchar(450) narrowing", + "the compatibility level is stated explicitly rather than inherited from the provider default", + "on EF Core 10, a first migration taken after switching to UseAzureSql is inspected for ALTER COLUMN statements that retype JSON columns", + "the application does not apply migrations from its own startup path against a cloud database" + ], + "probes": [ + { + "cmd": "PATH=\"$PATH:$HOME/.dotnet/tools\" dotnet ef migrations script --help | grep -q '^ -i|--idempotent'", + "where": "host", + "expect": { "succeeds": true }, + "why": "The flag SKILL.md tells a reader to run before an EF Core 10 upgrade, so the retype migration is read as SQL before it reaches a database. Read out of the tool's own help on 2026-09-03, dotnet ef 9.0.19. dotnet ef is a client tool a developer installs with dotnet tool install, never something an engine image ships, so this is a host probe." + }, + { + "cmd": "PATH=\"$PATH:$HOME/.dotnet/tools\" dotnet ef database update --help | grep -q -- '--connection '", + "where": "host", + "expect": { "succeeds": true }, + "why": "The whole permissions argument in SKILL.md rests on this flag existing: --connection is how a migration is applied by a deployment identity while the application keeps its own string in AddDbContext or OnConfiguring. Its help text on 9.0.19 reads 'Defaults to the one specified in AddDbContext or OnConfiguring.'" + }, + { + "cmd": "PATH=\"$PATH:$HOME/.dotnet/tools\" dotnet ef migrations --help | grep -q '^ has-pending-model-changes'", + "where": "host", + "expect": { "succeeds": true }, + "why": "The second of the three checks in the 'Check it worked' section. The command exists from EF Core 8 and is listed by dotnet ef 9.0.19; measured 2026-09-03, it exits 0 with 'No changes have been made to the model since the last migration.' and exits 1 after one property is added, which is what makes it usable in a pipeline." + }, + { + "cmd": "PATH=\"$PATH:$HOME/.dotnet/tools\" dotnet ef dbcontext --help | grep -q '^ info'", + "where": "host", + "expect": { "succeeds": true }, + "why": "The first check in 'Check it worked'. Measured 2026-09-03 on 9.0.19, dbcontext info prints 'Options: EngineType=AzureSql' under UseAzureSql and 'Options: EngineType=SqlServer' under UseSqlServer with the same connection string and the same provider name, so it is the only command that separates the two configurations." + }, + { + "sql": "CREATE TABLE dbo.probe_ef_indexed_string (Id int NOT NULL IDENTITY PRIMARY KEY, Name nvarchar(450) NOT NULL);\nCREATE INDEX IX_probe_ef_indexed_string_Name ON dbo.probe_ef_indexed_string (Name);\nINSERT INTO dbo.probe_ef_indexed_string (Name) VALUES (REPLICATE(N'a', 500));", + "expect": { "error": 2628 }, + "why": "The nvarchar(450) claim, made executable. An indexed string property with no HasMaxLength is generated as nvarchar(450), confirmed again on 2026-09-03 in a dotnet ef migrations script run, and the engine then refuses a longer value rather than trimming it. Msg 2628 is what makes the narrowing loud instead of silent, which is the correction this section carries." + }, + { + "sql": "CREATE TABLE dbo.probe_ef_idempotency (Id int NOT NULL IDENTITY PRIMARY KEY, IdempotencyKey uniqueidentifier NOT NULL, Amount decimal(10,2) NOT NULL);\nCREATE UNIQUE INDEX IX_probe_ef_idempotency_Key ON dbo.probe_ef_idempotency (IdempotencyKey);\nDECLARE @k uniqueidentifier = '00000000-0000-0000-0000-000000000001';\nINSERT INTO dbo.probe_ef_idempotency (IdempotencyKey, Amount) VALUES (@k, 10.00);\nINSERT INTO dbo.probe_ef_idempotency (IdempotencyKey, Amount) VALUES (@k, 10.00);", + "expect": { "error": 2601 }, + "why": "The first defence in 'Making the unit safe to run twice', at the storage layer where it actually lives. The second insert is the replayed delegate: with a unique index over the idempotency key it collides instead of duplicating. Microsoft Learn documents Msg 2601 as the unique index violation, distinct from 2627 for a UNIQUE constraint, and this probe creates an index." + }, + { + "sql": "CREATE TABLE dbo.probe_ef_nonunique (Id int NOT NULL IDENTITY PRIMARY KEY, IdempotencyKey uniqueidentifier NOT NULL);\nCREATE INDEX IX_probe_ef_nonunique_Key ON dbo.probe_ef_nonunique (IdempotencyKey);\nDECLARE @k uniqueidentifier = '00000000-0000-0000-0000-000000000002';\nINSERT INTO dbo.probe_ef_nonunique (IdempotencyKey) VALUES (@k);\nINSERT INTO dbo.probe_ef_nonunique (IdempotencyKey) VALUES (@k);\nSELECT COUNT(*) AS rows_after_replay FROM dbo.probe_ef_nonunique WHERE IdempotencyKey = @k;", + "expect": { "rows": [[2]] }, + "why": "The negative control for the probe above, and the duplicate write itself reduced to storage. Same key, same two inserts, only the index is no longer unique: both statements succeed with no error and the table holds two rows. Two here is the defect, not the absence of one, and it is why the skill says an idempotency key without a unique index is decorative." + }, + { + "sql": "SELECT CASE WHEN CURRENT_TIMEZONE() LIKE '%UTC%' THEN 1 ELSE 0 END AS timezone_is_utc,\n CASE WHEN ABS(DATEDIFF(second, GETDATE(), SYSUTCDATETIME())) <= 1 THEN 1 ELSE 0 END AS getdate_is_utc;", + "expect": { "rows": [[1, 1]] }, + "why": "The HasDefaultValueSql(\"GETDATE()\") claim. Measured 2026-08-28 the host was six hours behind the engine and GETDATE() still returned UTC, so a GETDATE() default behaves the same locally and in the cloud, and only a model carried over from an engine on local time changes meaning. A one second tolerance keeps this from failing on a clock boundary." + } + ] + }, + "triggering": { + "explicit": [ + "Use the ef-core-azure-sql skill", + "Set up EF Core against Azure SQL Database", + "Should I use UseAzureSql or UseSqlServer for my DbContext", + "Configure EnableRetryOnFailure for my Azure SQL DbContext", + "Wrap my EF Core transaction in an execution strategy so it can be retried" + ], + "implicit": [ + "The configured execution strategy SqlServerRetryingExecutionStrategy does not support user-initiated transactions", + "my DbContext worked fine until I wrapped two SaveChanges calls in a transaction", + "our EF Core service writes the same order twice whenever Azure SQL has a blip, and nothing is in the logs", + "is it safe to put BeginTransaction and Commit inside CreateExecutionStrategy().Execute", + "my EF Core app works locally but times out intermittently once it is pointed at Azure", + "why did my next EF migration try to ALTER every JSON column", + "my Include query returns tens of thousands of rows and is slow against Azure", + "does anything in my DbContext have to change when the app moves to a passwordless connection string" + ], + "routing": [ + "port my EF Core app from a self hosted database to Azure SQL Database", + "make my EF Core writes safe to retry against Azure SQL Database" + ], + "negative": [ + "Use Prisma against Azure SQL Database", + "Set up SQLAlchemy models for Azure SQL", + "Open a SqlConnection and run a query with Microsoft.Data.SqlClient", + "Size the connection pool and add transient retry underneath my .NET data access layer", + "Write a LINQ GroupBy query", + "Create an Entra ID application identity and grant it access to my database", + "Deploy my EF Core migrations from a release pipeline" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models deleted file mode 120000 index cb2ca2e..0000000 --- a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models +++ /dev/null @@ -1 +0,0 @@ -../../../skills/embeddings-and-external-models \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/SKILL.md b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/SKILL.md new file mode 100644 index 0000000..ea62add --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/SKILL.md @@ -0,0 +1,333 @@ +--- +name: embeddings-and-external-models +description: >- + Generates embeddings and chunks inside Azure SQL Database with CREATE EXTERNAL MODEL, + AI_GENERATE_EMBEDDINGS, AI_GENERATE_CHUNKS and sp_invoke_external_rest_endpoint, covering the + database scoped credential naming rule, the permissions, the dimension budget that decides which + embedding model fits, and the outbound allowlist. Use when someone asks to "create an external + model", "call AI_GENERATE_EMBEDDINGS", "embed text in T-SQL", "chunk text in the database", or + "call an Azure OpenAI endpoint from SQL"; when such a call fails on the credential secret, + managed identity, permissions, HTTPS or a blocked domain; and when embedding a whole table in + one statement runs for hours. This skill owns producing the vector and calling out of the + engine; storing and searching it is vector-search-azure-sql, the pipeline around it is + rag-on-azure-sql, and embedding offline is rag-local-with-container. +--- + +# Embeddings and external models in Azure SQL Database + +How the engine itself produces a vector, and how it is allowed to reach a model. Verified 2026-08-28 +against **both** a live Azure SQL Database (General Purpose serverless, compatibility level 170) and +the local container, using a **real Azure OpenAI deployment created for the run**. Option lists and +limits were re-sourced from Microsoft Learn on 2026-09-03. + +`AI_GENERATE_CHUNKS` splits text and `CREATE EXTERNAL MODEL` registers an endpoint; neither touches +the network. `AI_GENERATE_EMBEDDINGS` and `sp_invoke_external_rest_endpoint`, the call it builds +on, both do, and that is where everything below goes wrong. All four work in the container +as well as in the cloud: the 2026 catalog note calling this surface cloud only was wrong, and a real +embedding call completed from inside a container. + +## The correction + +**`CREATE EXTERNAL MODEL` validates almost nothing.** Measured on both engines, the DDL accepted an +`http://` location, a host no allowlist permits, a credential whose name can never match, and a +deployment that does not exist. All of it succeeded and appeared in `sys.external_models`. Each +failure surfaces at the first `AI_GENERATE_EMBEDDINGS`, naming an object rather than the broken +rule: + +| What is actually wrong | What the engine says | +|---|---| +| The location is `http://` | `Msg 31610, Accessing the external endpoint is only allowed via HTTPS` | +| The credential name is not a prefix of the location | `Msg 31630, The database scoped credential '' cannot be used` | +| A managed identity credential with no `SECRET` | `Msg 33047, Fail to obtain or decrypt secret for credential ''` | +| The caller lacks `EXECUTE` on the model | `Msg 15151, Cannot find the external model, or you do not have permission` | +| The domain is not on the cloud allowlist | `Msg 31612, Connections to the domain are not allowed` | +| The key is wrong, or absent | `Msg 31742, Unrecoverable HTTP error 401 occured` | + +**The create statement is not the test. The first call is.** Being wrong costs a pipeline that +deploys clean, passes every schema gate, and fails on the first row of the first ingest, pointing at +a credential, object or permission that is present and correct. + +## Which model fits, and it is not the best one + +A `vector` column holds at most **1998 dimensions** at the default `float32` base type. Learn's +Azure OpenAI model table gives the outputs: + +| Model | Output dimensions | Fits a vector column | +|---|---|---| +| `text-embedding-3-small` | 1536 | Yes | +| `text-embedding-ada-002` | 1536 | Yes, and it accepts no `dimensions` parameter | +| `text-embedding-3-large` | 3072 | **No, not at its default** | + +`text-embedding-3-large` is what an agent reaches for because Learn's benchmark table scores it +highest, and its 3072 dimension output fits no column this engine can declare. Only third generation +models accept `dimensions`, so the fix is one JSON property on the model, +`PARAMETERS = '{"dimensions":1536}'`, not a comment. Learn's `float16` base type doubles the ceiling +to 3996, on a page that excludes Azure SQL Database and behind `PREVIEW_FEATURES`, so do not design +around it. `vector-search-azure-sql` owns the column and the ceiling; this skill owns making the +model return a number that fits. + +## Step 1: chunk, which needs nothing but compatibility level 170 + +`AI_GENERATE_CHUNKS` runs with no endpoint, no credential and no permission: a user holding only +`db_datareader` ran it on both engines. Below compatibility level 170 Learn states the engine cannot +find the function, which reads as a missing feature rather than a setting. + +```sql +SELECT t.doc_id, c.chunk_set_id, c.chunk_order, c.chunk_offset, c.chunk +FROM dbo.documents AS t +CROSS APPLY AI_GENERATE_CHUNKS(source = t.body, chunk_type = FIXED, chunk_size = 400, + overlap = 10, enable_chunk_set_id = 1) AS c; +``` + +- **`overlap` is a percentage of `chunk_size`, not a character count.** Learn states a whole number + from 0 to 50, applied to `chunk_size`. Measured at `chunk_size = 40, overlap = 10` the second chunk + started at offset 37: four characters back, not ten. Read as characters it silently changes how + much context every chunk carries, and 50 is a hard ceiling. +- **`chunk_type = FIXED` is a keyword, and the only one.** `chunk_type = N'FIXED'` is `Msg 102`, and + so are `SENTENCE`, `PARAGRAPH`, `RECURSIVE`, `SEMANTIC`, `TOKEN` and `WORD`. Learn lists `FIXED` + alone, so semantic boundaries are work done outside this function. +- **`chunk_size` is characters, is required with `FIXED`, and splits mid word.** Measured at 40 the + second chunk began `azy dog. Azure SQL Database stores vecto`. It also keeps a chunk under the + model's 8192 token limit. +- **`enable_chunk_set_id = 1` is what makes a `CROSS APPLY` usable.** Without it `chunk_order` + restarts at 1 per row and nothing records which row a chunk came from. + +## Step 2: the credential, where the naming rule lives + +```sql +CREATE MASTER KEY ENCRYPTION BY PASSWORD = ''; -- once per database + +CREATE DATABASE SCOPED CREDENTIAL [https://.openai.azure.com/] +WITH IDENTITY = 'HTTPEndpointHeaders', SECRET = '{"api-key":""}'; +``` + +**The name of the credential is not a label. It is the match key.** Learn requires a valid URL on an +allowed domain, no query string, matching the called URL on protocol, fully qualified domain name +and every path segment, and more generic than it: a longest prefix match. So both +`https://.openai.azure.com/` and the fuller `.../openai/deployments//` work, while +`badcred` and a path segment the location lacks are both `Msg 31630`. That message reads like a +broken secret and is almost always the name instead. **The engine resolves the hostname before it +judges the name**, so an unresolvable host answers `Msg 31625` and hides the credential bug. Check +the prefix against the catalog offline, before regenerating any key. + +**Managed identity is the production identity, and needs a resource in the secret.** + +```sql +CREATE DATABASE SCOPED CREDENTIAL [https://.openai.azure.com/] +WITH IDENTITY = 'Managed Identity', SECRET = '{"resourceid":"https://cognitiveservices.azure.com"}'; +``` + +Measured in the cloud: with no `SECRET` the call failed with `Msg 33047, Fail to obtain or decrypt +secret`, which sounds like a master key problem and is not. Adding `resourceid` fixed it. The server +also needs an identity holding a role on the target resource, and without it the failure is an HTTP +401, indistinguishable from a wrong key. **On the container managed identity does not work at all**: +`Msg 31644, Server Managed Identity is disabled for this instance`, and its named remedy cannot run +there either. Which statement refuses it, measured 2026-09-06 on 18.0.226_4_147, is worth getting +right, because this skill named the wrong one until that day. `sp_configure` is not callable from a +user database at all, `Msg 2812, Could not find stored procedure`. Connected to `master` it runs and +stages the value happily. **`RECONFIGURE` is what refuses**, with +`Msg 40510, Statement 'CONFIG' is not supported in this version of SQL Server`, so the staged value +never takes effect and nothing tells you unless you read that far. Use a key locally and managed +identity in the cloud, changing only the credential. + +## Step 3: the model + +```sql +CREATE EXTERNAL MODEL text_embedder +WITH ( + LOCATION = 'https://.openai.azure.com/openai/deployments//embeddings?api-version=2024-10-21', + API_FORMAT = 'Azure OpenAI', + MODEL_TYPE = EMBEDDINGS, + MODEL = 'text-embedding-3-small', + CREDENTIAL = [https://.openai.azure.com/], + PARAMETERS = '{"dimensions":1536, "sql_rest_options": {"retry_count": 3}}' +); +``` + +- **`MODEL_TYPE = EMBEDDINGS` is the only value there is.** Learn lists no other; `CHAT`, + `COMPLETION`, `COMPLETIONS` and `RERANK` are all `Msg 102`. The engine makes a vector, never an + answer, so the generation half of a pipeline stays in the application. +- **`API_FORMAT` is required and is not validated against the location.** Learn's four values are + `Azure OpenAI`, `OpenAI`, `Ollama` and `ONNX Runtime`, the last a local runtime for a different + engine. The first three were accepted whatever the URL pointed at. Omitting it is `Msg 46505`, an + unlisted value `Msg 46508`. +- **`PARAMETERS` carries the dimension budget.** `'{"dimensions":768}'` returned a 768 dimension + vector from a model defaulting to 1536, measured; a value the model cannot produce is + `Msg 31742, Unrecoverable HTTP error 400`. +- **`PARAMETERS` also carries the retries.** `sql_rest_options.retry_count` takes 0 to 10 and retries + HTTP 408, 429, 500, 502, 503 and 504, honouring `Retry-After`. It defaults to 0, so a model without + it fails a whole batch on one rate limit response. +- **There is no `DROP EXTERNAL MODEL IF EXISTS`.** `Msg 156`; guard on `sys.external_models`. A + credential a model still uses cannot be dropped either, `Msg 46556`: drop the model first. + +## Step 4: permissions + +| Action | Permission | Failure when missing | +|---|---|---| +| Create a model, or alter someone else's | `CREATE EXTERNAL MODEL`, `ALTER ANY EXTERNAL MODEL` | `Msg 262` | +| `AI_GENERATE_EMBEDDINGS ... USE MODEL m` | `EXECUTE ON EXTERNAL MODEL::m` | `Msg 15151`, the object appears not to exist | +| `sp_invoke_external_rest_endpoint` | `EXECUTE ANY EXTERNAL ENDPOINT` | `Msg 8189` | +| Naming a credential on a direct REST call | `REFERENCES` on that credential | denied on the credential | +| `AI_GENERATE_CHUNKS` | none | it just runs | + +```sql +GRANT EXECUTE ON EXTERNAL MODEL::text_embedder TO [app_user]; +-- the two below only if the caller invokes REST directly +GRANT EXECUTE ANY EXTERNAL ENDPOINT TO [app_user]; +GRANT REFERENCES ON DATABASE SCOPED CREDENTIAL::[https://.openai.azure.com/] TO [app_user]; +``` + +**The `EXECUTE` denial is the one that wastes an afternoon.** A caller without it is told the model +"does not exist or you do not have permission", so the obvious move is to recreate it, which changes +nothing. + +**`Msg 15151` has a second cause that is not a permission at all**, measured on a logical server +2026-09-08: `CREATE EXTERNAL MODEL` and the `AI_GENERATE_EMBEDDINGS` that uses it **cannot be in the +same batch**. The model name binds when the batch compiles, before the create in that batch has run, +and there is no deferred name resolution for external models. Split by `GO` or run in two round trips +and the identical statements succeed. The message points at permissions the whole time. A caller going through `AI_GENERATE_EMBEDDINGS` needs neither of the other two grants, +because the model holds the credential: that is why to prefer it over raw REST. + +## Step 5: the allowlist, which is a cloud rule and not a local one + +**Azure SQL Database only calls hosts on a fixed allowlist**, and everything else is +`Msg 31612, Connections to the domain are not allowed`, raised before DNS or TLS. Learn's +`sp_invoke_external_rest_endpoint` page carries the list and is the contract, so read it there and +not from any table, this skill's included. Two things that list is not: + +- **Not "all of Azure".** Probed host by host in the cloud on 2026-08-28, `documents.azure.com`, + `management.azure.com` and `azureml.ms` were refused while `openai.azure.com` and + `blob.core.windows.net` were reached. Learn's route to a host off the list is API Management in + front of it, since `*.azure-api.net` is on it. +- **Not enforced on the container at all.** The same call to a public non-Azure host returned HTTP + 200 locally and `Msg 31612` in the cloud, so a pipeline proved locally can be refused on its first + cloud run by a policy no retry fixes. + +Re-probe rather than quote: call a host that does not exist under that domain. `Msg 31625` means +the domain is permitted; `Msg 31612` means it is not. + +Local hosting is closed off by two more container rules: a private address is `Msg 31624`, and a +certificate not chaining to a root the engine trusts is `Msg 31608`, a private authority in the +container's own trust store included, because the engine does not read that store. Learn's +`https://localhost:11435/api/embed` Ollama example is written for a differently hosted engine, so +embedding against a model server on the developer's machine is `rag-local-with-container`. + +## Step 6: embed, in batches, never in one statement + +```sql +UPDATE TOP (200) c + SET embedding = AI_GENERATE_EMBEDDINGS(c.chunk_text USE MODEL text_embedder), + embed_model = 'text-embedding-3-small', + embedded_at = SYSUTCDATETIME() + FROM dbo.document_chunks AS c + WHERE c.embedding IS NULL; +``` + +**One row is one HTTP round trip, and it does not parallelise.** One call took 295 ms and a 25 row +`UPDATE` took 8963 ms elapsed for 235 ms of CPU. So loop the statement above until it affects zero +rows: 200 rows is roughly a 70 second transaction, and the corpus is never one. Also: + +- **`NULL` input is an error, not a `NULL` result.** `Msg 8116, Argument data type NULL is invalid`. + Filter the nulls; an empty string is accepted and returns a real vector. +- **The returned dimension must match the target.** A 768 dimension result into a `vector(1536)` is + `Msg 42204`, the one loud failure in this surface. +- **A per call override exists** when a job needs a different dimension or retry budget than the + model carries: `AI_GENERATE_EMBEDDINGS(@t USE MODEL text_embedder PARAMETERS N'{"dimensions":768}')`. +- **Concurrency is capped**, at 10% of worker threads to a maximum of 150, and `Msg 10928` past it. + Measured 2026-09-08 on a Basic database: 70 concurrent sessions, 30 admitted and 40 refused with + `Resource ID : 1. The request limit for the database is 30 and has been reached`. It arrives at + **Level 20**, though `sys.messages` lists 10928 at severity 16, so a retry policy that filters on + severity does not see what the catalog predicts. + Learn's query for the per database number reads `sys.dm_user_db_resource_governance`, cloud only + and `Msg 208` on the container, so size parallelism in the cloud or not at all. +- Record which model produced each vector. The reason belongs to `rag-on-azure-sql`. + +## Step 7: the general REST call, when no model type fits + +The escape hatch, under the same allowlist, HTTPS and certificate rules. + +```sql +DECLARE @response NVARCHAR(MAX), @return INT; +EXEC @return = sp_invoke_external_rest_endpoint + @url = 'https://.openai.azure.com/openai/deployments//embeddings?api-version=2024-10-21', + @method = 'POST', @payload = @body, @headers = N'{"Accept":"application/json"}', + @timeout = 60, @retry_count = 3, + @credential = [https://.openai.azure.com/], + @response = @response OUTPUT; + +SELECT @return AS http_status, JSON_VALUE(@response, '$.response.status.http.code') AS code; +``` + +- **`@timeout` defaults to 30 seconds**, accepts 1 to 230, and becomes the cumulative budget once + `@retry_count` is set; a slow endpoint fails on that default first. +- **`@return` is 0 for a 2xx and otherwise the HTTP status code.** Only a call that could not be made + at all throws, so a procedure never reading `@return` treats a 429 as success. An unparseable `200` + body is `Msg 11558`, which looks like a query bug and is a content type mismatch. + +## Check it worked + +Run these in the target database in order: each fails faster than the next. + +```sql +-- 1. The model is registered. Expect one row per model, with the dimensions +-- you set visible in the parameters column, without parsing any DDL. +SELECT name, api_format, model_type_desc, model, parameters FROM sys.external_models; + +-- 2. The credential name is a prefix of the location. The engine will not +-- report this until the host resolves, so check it here. Expect 'prefix ok'. +SELECT m.name, c.name AS credential, + CASE WHEN c.name IS NULL THEN 'no credential, expect HTTP 401' + WHEN LEFT(m.location, LEN(c.name)) = c.name THEN 'prefix ok' + ELSE 'WILL FAIL WITH 31630' END AS naming +FROM sys.external_models AS m +LEFT JOIN sys.database_scoped_credentials AS c ON c.credential_id = m.credential_id; + +-- 3. The application user holds EXECUTE and nothing wider. Expect one +-- EXTERNAL_MODEL row for it and no CREATE EXTERNAL MODEL row. +SELECT class_desc, permission_name, USER_NAME(grantee_principal_id) AS grantee +FROM sys.database_permissions +WHERE permission_name IN ('EXECUTE', 'CREATE EXTERNAL MODEL', 'EXECUTE ANY EXTERNAL ENDPOINT'); + +-- 4. Last, because it is the only one that leaves the engine: the endpoint +-- answers and its dimension matches the column. Expect one row reading +-- 1536; a mismatch prints nothing and fails on Msg 42204, as it should. +DECLARE @v vector(1536) = AI_GENERATE_EMBEDDINGS(N'probe text' USE MODEL text_embedder); +SELECT COUNT(*) AS dimensions FROM OPENJSON(CAST(@v AS nvarchar(max))); +``` + +Run the file rather than pasting it, so a failure stops a pipeline instead of scrolling by: + +```bash +# cloud, Microsoft Entra; use -U and -P against the container instead of -G +sqlcmd -S "$SQL_SERVER" -d "$SQL_DB" -G -b -m-1 -i check-embeddings.sql +``` + +`-b` sets a non-zero exit only at severity 11 and above, and this surface emits severity 10 messages: +`Msg 31616`, the overwritten `User-Agent` header, is one. Without `-m-1` those print their text with +no `Msg` number and the run looks clean, so use both. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build `Msg 31616` never appears at all and a run that +overwrote the header still looks clean. Run the file through the ODBC build, or read the number out +of `sys.messages` on a connection that works. `build-app-on-azure-sql` tells the two builds apart in +one table. + +## References + +- [references/measured-external-model.md](references/measured-external-model.md): open it when a + claim above needs re-verifying, or for the naming, permission, allowlist and certificate matrices + and the statements reproducing them. +- [CREATE EXTERNAL MODEL](https://learn.microsoft.com/sql/t-sql/statements/create-external-model-transact-sql): + read before changing `API_FORMAT`, `PARAMETERS` or `retry_count`. +- [AI_GENERATE_EMBEDDINGS](https://learn.microsoft.com/sql/t-sql/functions/ai-generate-embeddings-transact-sql): + read for the extended events to turn on when a call fails without saying why. +- [AI_GENERATE_CHUNKS](https://learn.microsoft.com/sql/t-sql/functions/ai-generate-chunks-transact-sql): + read before choosing a chunk type, the part of this surface most likely to grow. +- [sp_invoke_external_rest_endpoint](https://learn.microsoft.com/sql/relational-databases/system-stored-procedures/sp-invoke-external-rest-endpoint-transact-sql): + read before depending on any host, for the current allowlist and the payload and header limits. +- `vector-search-azure-sql`: storing and searching the vector this skill produces. +- `rag-on-azure-sql`: the pipeline this call sits inside, and why provenance matters. +- `rag-local-with-container` and `azuresql-db-rag`: embedding offline from application code, where + this in-database path is not available. diff --git a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/references/measured-external-model.md b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/references/measured-external-model.md new file mode 100644 index 0000000..5b5cdae --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/references/measured-external-model.md @@ -0,0 +1,258 @@ +# Measured behaviour of the external model surface, 2026-08-28 + +## Contents + +- [How this was run](#how-this-was-run) +- [What the DDL accepts without complaint](#what-the-ddl-accepts-without-complaint) +- [The credential naming matrix](#the-credential-naming-matrix) +- [Managed identity](#managed-identity) +- [The permission matrix](#the-permission-matrix) +- [The cloud allowlist, probed host by host](#the-cloud-allowlist-probed-host-by-host) +- [Outbound rules measured on the container](#outbound-rules-measured-on-the-container) +- [AI_GENERATE_CHUNKS](#ai_generate_chunks) +- [AI_GENERATE_EMBEDDINGS](#ai_generate_embeddings) +- [Cost and concurrency](#cost-and-concurrency) +- [Claims that did not hold](#claims-that-did-not-hold) +- [Reproducing this](#reproducing-this) + +## How this was run + +Two engines, the same statements, on the same day. + +- **Cloud.** An Azure SQL Database provisioned for the run: General Purpose serverless, two vCores, + compatibility level 170, Microsoft Entra authentication only, `SERVERPROPERTY('EngineEdition')` + returning 5. Deleted after the run. +- **Local.** The Azure SQL Database container, product version 12.0.2000.8, also reporting + `EngineEdition` 5. + +An Azure OpenAI resource with a `text-embedding-3-small` deployment was created for the run and +deleted after it, so the embedding calls below are real calls that returned real vectors, not +placeholder failures. + +Where the two engines agreed, the result is stated once. Every disagreement is called out. + +## What the DDL accepts without complaint + +All of these `CREATE EXTERNAL MODEL` statements **succeeded** on both engines and appeared in +`sys.external_models`. None of them can ever produce an embedding. + +| Statement | Why it can never work | +|---|---| +| `LOCATION = 'http://host:11434/api/embed'` | Plain HTTP is refused at call time | +| `LOCATION = 'https://api.github.com/rate_limit'`, `API_FORMAT = 'OpenAI'` | Not an embedding endpoint, and blocked in the cloud | +| `API_FORMAT = 'Ollama'` against an Azure OpenAI URL | Format is never compared with the location | +| `CREDENTIAL = badcred` where the name is not a URL prefix | The credential can never be selected | +| A deployment name that does not exist | Discovered as an HTTP 404 at call time | + +What the DDL **does** reject: + +| Statement | Error | +|---|---| +| `API_FORMAT` omitted | `Msg 46505, Missing required external DDL option 'API_FORMAT'` | +| `API_FORMAT = 'NotAFormat'` | `Msg 46508, Incorrect syntax on external DDL option 'API_FORMAT'` | +| `MODEL_TYPE = CHAT`, `COMPLETION`, `COMPLETIONS`, `RERANK` | `Msg 102`. `EMBEDDINGS` is the only value | +| `DROP EXTERNAL MODEL IF EXISTS m` | `Msg 156, Incorrect syntax near the keyword 'IF'` | +| Dropping a credential still referenced by a model | `Msg 46556, Cannot drop the credential '' because it is used by an external model` | + +`sys.external_models` carries `location`, `api_format`, `model_type_desc`, `model`, `credential_id` +and a `parameters` column of type `json`, so everything above is readable back without parsing DDL. + +## The credential naming matrix + +Everything else held constant: a real endpoint, a real key, a real deployment. + +| Credential name | Model `LOCATION` | Result | +|---|---|---| +| `https://.openai.azure.com/` | `https://.openai.azure.com/openai/deployments//embeddings?api-version=...` | Embedding returned | +| `https://.openai.azure.com/openai/deployments//` | the same | Embedding returned | +| `badcred` | the same | `Msg 31630` | +| `micred` | the same | `Msg 31630` | +| `https://.openai.azure.com/mi/` | a location with no `/mi/` segment | `Msg 31630` | +| `https://.openai.azure.com/openai/deployments//` with a deliberately wrong key | the same | `Msg 31742, Unrecoverable HTTP error 401` | +| no `CREDENTIAL` clause at all | the same | `Msg 31742, Unrecoverable HTTP error 401` | + +The last two rows are what makes the rule testable. A wrong **name** never reaches the network and +gives `Msg 31630`. A wrong **key** reaches the network and gives a 401. If the message is 31630 the +key is irrelevant. + +`CREATE MASTER KEY ENCRYPTION BY PASSWORD` was required once per database before the first +credential on both engines. + +## Managed identity + +| Attempt | Cloud | Container | +|---|---|---| +| `IDENTITY = 'Managed Identity'` with no `SECRET` | `Msg 33047, Fail to obtain or decrypt secret for credential ''` | `Msg 31644` | +| `IDENTITY = 'Managed Identity'`, `SECRET = '{"resourceid":"https://cognitiveservices.azure.com"}'` | **Embedding returned** | `Msg 31644, Server Managed Identity is disabled for this instance of SQL Server. Use sp_configure 'allow server scoped db credentials' to enable it` | +| The `sp_configure` the container's message names | not applicable | `Msg 40510, Statement 'CONFIG' is not supported in this version of SQL Server` | + +The cloud path additionally needed a system assigned identity on the logical server and a role +assignment for that identity on the target resource. Without the role the failure is an HTTP 401, +not a credential error, which is the same symptom as a wrong key. + +The container row is a hard parity break: the error names a remedy the container cannot execute. + +## The permission matrix + +Tested with a login and a database user holding `db_datareader` and `db_datawriter` and nothing +else, then re-tested after each grant. + +| Action | Without the permission | Permission that fixes it | +|---|---|---| +| `CREATE EXTERNAL MODEL` | `Msg 262, CREATE EXTERNAL MODEL permission denied in database ''` | `GRANT CREATE EXTERNAL MODEL TO ` | +| `AI_GENERATE_EMBEDDINGS ... USE MODEL m` | `Msg 15151, Cannot find the external model 'm', because it does not exist or you do not have permission` | `GRANT EXECUTE ON EXTERNAL MODEL::m TO ` | +| `sp_invoke_external_rest_endpoint` | `Msg 8189, You do not have permission to run 'sys.sp_invoke_external_rest_endpoint'` | `GRANT EXECUTE ANY EXTERNAL ENDPOINT TO ` | +| `AI_GENERATE_CHUNKS` | ran successfully with no grant at all | none | + +`sys.fn_builtin_permissions(DEFAULT)` lists `CREATE EXTERNAL MODEL`, `ALTER ANY EXTERNAL MODEL` and +`EXECUTE ANY EXTERNAL ENDPOINT` as database permissions. +`sys.fn_builtin_permissions('EXTERNAL MODEL')` lists `VIEW DEFINITION`, `ALTER`, `TAKE OWNERSHIP`, +`EXECUTE` and `CONTROL` as the object level set. + +After the grants, `sys.database_permissions` showed `CREATE EXTERNAL MODEL` and +`EXECUTE ANY EXTERNAL ENDPOINT` at class `DATABASE` and `EXECUTE` at class `EXTERNAL_MODEL`. + +## The cloud allowlist, probed host by host + +Method: call `sp_invoke_external_rest_endpoint` against `https://zzz-not-real./`. A domain +on the allowlist gets far enough to attempt a DNS lookup and fails with +`Msg 31625, DNS resolution of the hostname has failed with windows sockets error 11001`. A domain +that is not on it fails earlier with `Msg 31612`. + +| Domain probed | Cloud result | +|---|---| +| `openai.azure.com` | allowed, reached DNS | +| `cognitiveservices.azure.com` | allowed, reached DNS | +| `services.ai.azure.com` | allowed, reached DNS | +| `inference.ai.azure.com` | allowed, reached DNS | +| `search.windows.net` | allowed, reached DNS | +| `vault.azure.net` | allowed, reached DNS | +| `blob.core.windows.net` | allowed, reached DNS | +| `queue.core.windows.net` | allowed, reached DNS | +| `table.core.windows.net` | allowed, reached DNS | +| `servicebus.windows.net` | allowed, reached DNS | +| `eventgrid.azure.net` | allowed, reached DNS | +| `azure-api.net` | allowed, reached DNS | +| `azurewebsites.net` | allowed, reached DNS | +| `documents.azure.com` | **`Msg 31612`** | +| `management.azure.com` | **`Msg 31612`** | +| `azureml.ms` | **`Msg 31612`** | +| `api.github.com` | **`Msg 31612`** | + +Reaching DNS is evidence the domain is permitted, not evidence a real host under it will answer. +Treat this table as a snapshot of a policy that is expected to change, and re-probe rather than +quoting it. + +## Outbound rules measured on the container + +The same probes on the container behave differently, and the difference is not a subset +relationship. + +| Probe | Container result | +|---|---| +| `sp_invoke_external_rest_endpoint` to `https://api.github.com/rate_limit` | **HTTP 200 with the full response body.** No allowlist | +| A `200` response whose body is not JSON | `Msg 11558, The @result JSON string could not be parsed` | +| A supplied `User-Agent` header | replaced, with a warning that says so | +| `http://` location | `Msg 31610, Accessing the external endpoint is only allowed via HTTPS` | +| A hostname resolving to a private address | `Msg 31624, Connection to the external endpoint IP is not allowed. URL contains a hostname that is resolved to a blocked IP` | +| A self signed certificate | `Msg 31608, HRESULT 0x80070008` | +| A certificate from an untrusted root | `Msg 31608, HRESULT 0x80070008` | +| An expired certificate | `Msg 31608, HRESULT 0x80070020` | +| A valid publicly trusted certificate | the call completes | + +The certificate row was pinned down rather than assumed. A private certificate authority was +generated, installed into the container's operating system trust store with that distribution's own +tooling, and confirmed accepted by a command line client running inside the same container. The +engine still returned `Msg 31608`, before and after a restart. **The engine does not use the +container's trust store**, so there is no supported way to point an external model at a locally +hosted endpoint. + +## AI_GENERATE_CHUNKS + +Identical on both engines, including the offsets. + +```sql +SELECT chunk, chunk_order, chunk_offset, chunk_length +FROM AI_GENERATE_CHUNKS(source = N'The quick brown fox jumps over the lazy dog. Azure SQL Database stores vectors natively.', + chunk_type = FIXED, chunk_size = 40, overlap = 10); +``` + +| chunk | chunk_order | chunk_offset | chunk_length | +|---|---|---|---| +| `The quick brown fox jumps over the lazy ` | 1 | 1 | 40 | +| `azy dog. Azure SQL Database stores vecto` | 2 | 37 | 40 | +| `ectors natively.` | 3 | 73 | 16 | + +- `chunk_type = N'FIXED'` quoted: `Msg 102, Incorrect syntax near 'FIXED'`. +- `SENTENCE`, `PARAGRAPH`, `RECURSIVE`, `SEMANTIC`, `TOKEN`, `WORD`: all `Msg 102` on both engines. +- Sizes and overlap are characters. Word boundaries are not respected. + +## AI_GENERATE_EMBEDDINGS + +Against the real deployment, both engines returned a vector. + +| Input | Result | +|---|---| +| `N'the quick brown fox'`, default model dimensions | 1536 dimensions, base type `float32` | +| the same model with `PARAMETERS = '{"dimensions":512}'` | 512 dimensions | +| the same model with `PARAMETERS = '{"dimensions":768}'` | 768 dimensions | +| `PARAMETERS = '{"dimensions":9999}'` | `Msg 31742, Unrecoverable HTTP error 400` | +| `NULL` | `Msg 8116, Argument data type NULL is invalid for argument 2 of ai_generate_embeddings function` | +| `N''` | a real vector | +| a 768 dimension result assigned to `VECTOR(1536)` | `Msg 42204, The vector dimensions 1536 and 768 do not match` | + +The cloud and container vectors for the same input agreed to roughly four decimal places, which is +the embedding service's own non-determinism rather than an engine difference. + +## Cost and concurrency + +Measured on the container against the real endpoint, with `SET STATISTICS TIME ON`. + +| Statement | Elapsed | CPU | +|---|---|---| +| One `AI_GENERATE_EMBEDDINGS` call | 295 ms | 7 ms | +| `UPDATE` embedding 25 rows in one statement | 8963 ms | 235 ms | + +25 rows at 358 ms each is the single call time repeated. There is no batching and no parallelism +inside the statement, so a one million row corpus is roughly one hundred hours in a single +transaction. Batch it, and drive the batches from a filtered index on the unembedded rows. + +## Claims that did not hold + +1. **The catalog note said this surface was cloud only and unavailable in the container.** Wrong on + every count. All four objects exist locally, and a real embedding call completed from inside a + container. +2. **The natural reading of `Msg 31630` is that the secret is broken.** It is almost always the + credential's name not being a URL prefix of the location. That distinction is not in the message. +3. **The natural reading of `Msg 33047` is a database master key problem.** Measured, it is a + managed identity credential missing its `resourceid` secret, and the master key was present and + working for other credentials in the same database. +4. **The natural reading of `Msg 15151` is that the model was never created.** It is also what a + missing `EXECUTE` grant looks like, and the object is right there in `sys.external_models`. +5. **"Azure endpoints are allowed" is not the rule.** The resource manager and Cosmos DB endpoints + are refused, and something outside `azure.com` entirely is allowed. +6. **The container is not a subset of the cloud here.** It permits outbound calls the cloud refuses + and refuses an identity the cloud accepts. + +## Reproducing this + +The allowlist probe is the cheapest of these to re-run and the one most likely to have moved: + +```sql +DECLARE @response NVARCHAR(MAX), @return INT; +BEGIN TRY + EXEC @return = sp_invoke_external_rest_endpoint + @url = 'https://zzz-not-real.openai.azure.com/', @method = 'GET', + @response = @response OUTPUT; + SELECT 'reached the network' AS verdict; +END TRY +BEGIN CATCH + SELECT ERROR_NUMBER() AS error_number, ERROR_MESSAGE() AS message; +END CATCH; +``` + +`31625` means the domain is permitted. `31612` means it is not. Swap the host and repeat. + +The credential naming rule reproduces with two credentials over one endpoint: one named for the +endpoint URL and one named anything else, both holding the same key. The first returns a vector and +the second returns `Msg 31630`. diff --git a/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/skill.spec.jsonc new file mode 100644 index 0000000..d8beb60 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/embeddings-and-external-models/skill.spec.jsonc @@ -0,0 +1,273 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "embeddings-and-external-models", + "domain": "ai-vector-and-rag", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize", "multi-step-with-silent-failures"], + "correction": "Agents believe CREATE EXTERNAL MODEL is the step that proves an embedding endpoint works, and they believe a database scoped credential's name is a label. Both are wrong, and measured on 2026-08-28 against a live Azure SQL Database, the local Azure SQL Database container and a real Azure OpenAI deployment created for the run, the DDL accepted an http location, a domain no allowlist permits, a credential whose name can never match and a deployment that does not exist, all of it succeeding and appearing in sys.external_models. Every one of those failures arrives at the first AI_GENERATE_EMBEDDINGS wearing the name of an object rather than the rule that was broken: the credential name has to be a URL prefix of the model's LOCATION or the call fails with Msg 31630 saying the credential cannot be used, which reads as a broken secret; a managed identity credential without SECRET '{\"resourceid\":\"https://cognitiveservices.azure.com\"}' fails with Msg 33047 about failing to decrypt a secret, which reads as a master key problem; a missing EXECUTE ON EXTERNAL MODEL grant fails with Msg 15151 saying the model does not exist, which reads as a missing object. MODEL_TYPE = EMBEDDINGS is the only value that exists, so the generation half of a pipeline cannot live in the engine. AI_GENERATE_CHUNKS supports only chunk_type = FIXED, as an unquoted keyword, sized in characters and splitting mid word. AI_GENERATE_EMBEDDINGS is one serial HTTP round trip per row, measured at 295 ms for one call and 8963 ms elapsed for a 25 row UPDATE, so the set based statement an agent writes first is a multi hour transaction. And Azure SQL Database only calls hosts on a fixed allowlist, refusing everything else with Msg 31612 before DNS, including the resource manager and Cosmos DB endpoints, while the container enforces no allowlist at all and happily returns HTTP 200 from a public non-Azure host. Being wrong costs deployment that passes every gate and fails on the first row of the first ingest, hours spent regenerating a key that was never the problem, an ingest that holds one transaction open for the length of the corpus, and a pipeline proved locally that is refused by policy on its first cloud run.", + "value_probe": { + "gate": "The answer locates the failure in the relationship between the credential's name and the model's LOCATION rather than in the contents of the secret, and reworks the ingest into bounded batches instead of one whole corpus statement.", + "task": "We are adding search to a document store on Azure SQL Database and we want the engine itself to do the embedding so there is no extra service to operate. All of this ran clean:\n\n```sql\nCREATE DATABASE SCOPED CREDENTIAL [AzureOpenAICred]\n WITH IDENTITY = 'HTTPEndpointHeaders', SECRET = '{\"api-key\":\"REDACTED\"}';\n\nCREATE EXTERNAL MODEL DocEmbedder\n WITH (LOCATION = 'https://our-aoai.openai.azure.com/openai/deployments/text-embedding-3-large/embeddings?api-version=2024-06-01',\n API_FORMAT = 'Azure OpenAI',\n MODEL_TYPE = EMBEDDINGS,\n MODEL = 'text-embedding-3-large',\n CREDENTIAL = [AzureOpenAICred]);\n```\n\nThe model shows up in sys.external_models, but the first ingest run fails saying the credential cannot be used. We have already rotated the key twice and it made no difference. The plan for tonight was to chunk each document at 500 characters with 50 of overlap and then run the whole 60000 row corpus in a single UPDATE while nobody is working.\n\nTell us what is wrong and what the ingest should actually look like.", + "states": [ + "The credential's name has to be a URL prefix of the model's LOCATION, which is why this particular call is refused.", + "Creating the external model validates nothing about the endpoint, the deployment or the credential, so every one of these mistakes surfaces at the first embedding call rather than at deployment.", + "The failure message names an object rather than the rule that was broken, which is why rotating the key was never going to help.", + "The overlap argument is a percentage of the chunk size rather than a number of characters, so 50 with a chunk size of 500 does not overlap fifty characters.", + "Embedding is one serial round trip per row, so a whole corpus statement is a single very long transaction and the work has to be batched.", + "The chosen model emits more numbers per row than a vector column can hold, so it cannot be stored as one vector column." + ], + "refutes": [ + "A database scoped credential's name is an arbitrary label, so this failure is a bad, expired or wrongly pasted key.", + "The overlap argument is a character count, so 50 with a chunk size of 500 gives a fifty character overlap.", + "A model emitting 3072 numbers per row can be stored directly in an Azure SQL Database vector column.", + "Successful creation of the external model proves the endpoint, the deployment and the credential are correct and reachable.", + "Embedding the whole corpus in one set based UPDATE is fine because the engine issues the calls in parallel.", + "The engine can be pointed at any embedding endpoint that is reachable over the internet, with no restriction on the host." + ] + }, + "posture": ["read", "write", "execute", "admin"], + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "vector-search-azure-sql", + "rag-on-azure-sql", + "rag-local-with-container", + "langchain-and-llamaindex-on-azure-sql", + "chunking-strategies", + "entra-id-auth", + "connect-to-azure-sql", + "azuresql-db-rag" + ], + "validation": { + "target": "both", + "setup": [ + "a database with permission to create a master key, a database scoped credential and an external model", + "an embedding deployment reachable on a domain the target engine is allowed to call", + "a second low privilege database user, for the permission checks" + ], + "assert": [ + "every external model was called once successfully immediately after it was created", + "the database scoped credential's name is a URL prefix of the external model's LOCATION", + "the production credential uses managed identity with a resourceid secret, and the server identity holds a role on the target resource", + "no key, endpoint host name or resource name appears in source control", + "the dimension requested in PARAMETERS equals the declared column dimension", + "embedding runs in bounded batches filtered to rows with no vector, and no statement embeds a whole table", + "the application caller holds EXECUTE on the model and does not hold CREATE EXTERNAL MODEL", + "every endpoint the design depends on was checked against the allowlist from a cloud database rather than from the container", + "chunking uses chunk_type = FIXED with chunk_size stated in characters and overlap stated as a percentage of it" + ], +"probes": [ + { + "sql": "SELECT chunk_order, chunk_offset, chunk_length\nFROM AI_GENERATE_CHUNKS(source = N'The quick brown fox jumps over the lazy dog. Azure SQL Database stores vectors natively.',\n chunk_type = FIXED, chunk_size = 40, overlap = 10);", + "expect": { + "rows": [ + [ + 1, + 1, + 40 + ], + [ + 2, + 37, + 40 + ], + [ + 3, + 73, + 16 + ] + ] + }, + "why": "The defect this rewrite found, made executable. The skill previously said overlap was a character count and its own measurement contradicted it. Microsoft Learn's AI_GENERATE_CHUNKS page states OVERLAP is a percentage of CHUNK_SIZE, a whole number from 0 to 50. At chunk_size 40 and overlap 10 the second chunk must start at offset 37, four characters back, not ten: 10% of 40. If overlap were characters the second offset would be 31. The third row also pins the mid-word split and the ragged final chunk of an 88 character source. Measured on both engines 2026-08-28." + }, + { + "sql": "SELECT chunk FROM AI_GENERATE_CHUNKS(source = N'abcdefgh', chunk_type = N'FIXED', chunk_size = 2);", + "expect": { + "error": 102 + }, + "why": "The one-character difference an agent writes first, because every other string-valued option in T-SQL is quoted. Learn's syntax block gives CHUNK_TYPE = FIXED as a bare keyword and lists FIXED as the only accepted value. Msg 102 is this skill's measurement of 2026-08-28, identical on both engines, and the same number comes back for SENTENCE, PARAGRAPH, RECURSIVE, SEMANTIC, TOKEN and WORD." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_chat\nWITH (LOCATION = 'https://zzz-not-real.openai.azure.com/openai/deployments/d/chat/completions?api-version=2024-10-21',\n API_FORMAT = 'Azure OpenAI', MODEL_TYPE = CHAT, MODEL = 'gpt-4o');", + "expect": { + "error": 102 + }, + "why": "The boundary of the whole feature, asserted rather than described. Learn's CREATE EXTERNAL MODEL page lists EMBEDDINGS as the only accepted MODEL_TYPE, so the engine can produce a vector and never an answer, and the generation half of a retrieval pipeline cannot move into T-SQL. Msg 102 is this skill's measurement of 2026-08-28 for CHAT, COMPLETION, COMPLETIONS and RERANK alike. If this ever succeeds the skill's scope boundary against rag-on-azure-sql has moved." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_noformat\nWITH (LOCATION = 'https://zzz-not-real.openai.azure.com/openai/deployments/d/embeddings?api-version=2024-10-21',\n MODEL_TYPE = EMBEDDINGS, MODEL = 'text-embedding-3-small');", + "expect": { + "error": 46505 + }, + "why": "API_FORMAT is the one option the DDL genuinely enforces, which is what makes the rest of the correction surprising: everything the create statement CAN check it does check, and it can check almost nothing. Msg 46505, missing required external DDL option, is this skill's measurement of 2026-08-28. Learn documents API_FORMAT as required and gives four accepted values." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_ddl\nWITH (LOCATION = 'http://host.docker.internal:11434/api/embed',\n API_FORMAT = 'Ollama', MODEL_TYPE = EMBEDDINGS, MODEL = 'nomic-embed-text');\nGO\nSELECT COUNT(*) AS created FROM sys.external_models WHERE name = 'probe_em_ddl';", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The skill's headline correction, made executable: CREATE EXTERNAL MODEL validates almost nothing. This statement names a plain-HTTP scheme the engine refuses at call time, a host that resolves to a private address, no credential at all, and an API_FORMAT that is never compared against the location. It succeeds anyway and the model appears in sys.external_models. A create that returns cleanly is therefore not evidence of anything, which is the assumption this skill exists to correct." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_http\nWITH (LOCATION = 'http://host.docker.internal:11434/api/embed',\n API_FORMAT = 'Ollama', MODEL_TYPE = EMBEDDINGS, MODEL = 'nomic-embed-text');\nGO\nSELECT AI_GENERATE_EMBEDDINGS(N'probe text' USE MODEL probe_em_http);", + "expect": { + "error": 31610 + }, + "why": "The same model as the probe above, called rather than created, and the pair is the point: identical DDL, one succeeds and one fails. Learn states on both the CREATE EXTERNAL MODEL and sp_invoke_external_rest_endpoint pages that only HTTPS with TLS is supported; Msg 31610 is this skill's measurement of 2026-08-28. The location is the real Ollama port on the host running this lane, so the probe also settles the question an agent asks next: an embedding model served on the developer's own machine over plain HTTP is refused by the engine before any network call, which is why local embedding belongs to rag-local-with-container and not here." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##')\n CREATE MASTER KEY ENCRYPTION BY PASSWORD = N'Pr0be-N0t-A-Real-P@ssw0rd';\nGO\nCREATE DATABASE SCOPED CREDENTIAL [badcred_dns]\nWITH IDENTITY = 'HTTPEndpointHeaders', SECRET = '{\"api-key\":\"not-a-real-key\"}';\nGO\nCREATE EXTERNAL MODEL probe_em_dns\nWITH (LOCATION = 'https://zzz-not-real.openai.azure.com/openai/deployments/d/embeddings?api-version=2024-10-21',\n API_FORMAT = 'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = 'text-embedding-3-small',\n CREDENTIAL = [badcred_dns]);\nGO\nSELECT AI_GENERATE_EMBEDDINGS(N'probe text' USE MODEL probe_em_dns);", + "expect": { + "error": 31625 + }, + "why": "Which failure the engine reports when two things are wrong at once, and it is not the one an author expects. Microsoft Learn's error table gives 31625 as 'DNS resolution of the hostname has failed with windows sockets error %d' and 31630 as 'The database scoped credential cannot be used to invoke an external rest endpoint': different stages, not synonyms. This fixture breaks both, a credential name that is not a URL and so can never match, and a host under an allowed domain that does not exist. Measured 2026-09-03 on the container, EngineEdition 5, ProductVersion 12.0.2000.8, the answer is 31625, while the 2026-08-28 run with the same shape of bad credential against a real resolvable endpoint returned 31630. So the hostname is resolved before the credential name is judged, and a credential naming bug stays invisible until the endpoint is real. An earlier revision of this probe asserted 31630 here: it took the host from this skill's allowlist method, where a non-existent name is the whole point, and the number from its credential matrix, where the endpoint was real. That is why the probe below checks the naming rule offline instead." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##')\n CREATE MASTER KEY ENCRYPTION BY PASSWORD = N'Pr0be-N0t-A-Real-P@ssw0rd';\nGO\nCREATE DATABASE SCOPED CREDENTIAL [badcred_prefix]\nWITH IDENTITY = 'HTTPEndpointHeaders', SECRET = '{\"api-key\":\"not-a-real-key\"}';\nGO\nCREATE EXTERNAL MODEL probe_em_naming\nWITH (LOCATION = 'https://zzz-not-real.openai.azure.com/openai/deployments/d/embeddings?api-version=2024-10-21',\n API_FORMAT = 'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = 'text-embedding-3-small',\n CREDENTIAL = [badcred_prefix]);\nGO\nSELECT CASE WHEN LEFT(m.location, LEN(c.name)) = c.name THEN 'prefix ok'\n ELSE 'WILL FAIL WITH 31630' END AS naming\nFROM sys.external_models AS m\nJOIN sys.database_scoped_credentials AS c ON c.credential_id = m.credential_id\nWHERE m.name = 'probe_em_naming';", + "expect": { + "rows": [ + [ + "WILL FAIL WITH 31630" + ] + ] + }, + "why": "The credential naming rule pinned where it can actually be pinned. The probe above shows the engine will not report 31630 until the hostname resolves, so on a lane with no reachable endpoint the rule is unprovable by calling it. This is the check the skill's own Check it worked section ships, run against a model the DDL accepted happily: Learn requires the credential name to be a valid URL matching the called URL on protocol, fully qualified domain name and every path segment, and 'badcred_prefix' is not a URL at all. The query catches that from sys.external_models and sys.database_scoped_credentials with no network call, which is the point being made: the prefix is checkable offline and should be checked before anyone regenerates a key. If this row ever reads 'prefix ok' the shipped check query has stopped detecting the failure it exists to detect." + }, + { + "sql": "DROP EXTERNAL MODEL IF EXISTS probe_em_never_created;", + "expect": { + "error": 156 + }, + "why": "The guard clause every deployment script reaches for, and it does not exist for this object type. Msg 156, incorrect syntax near the keyword IF, is this skill's measurement of 2026-08-28 on both engines, and Learn's CREATE EXTERNAL MODEL page shows only a bare DROP EXTERNAL MODEL. A rerunnable script has to test sys.external_models instead, and has to drop the model before the credential it uses." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_null\nWITH (LOCATION = 'https://zzz-not-real.openai.azure.com/openai/deployments/d/embeddings?api-version=2024-10-21',\n API_FORMAT = 'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = 'text-embedding-3-small');\nGO\nSELECT AI_GENERATE_EMBEDDINGS(NULL USE MODEL probe_em_null);", + "expect": { + "error": 8116 + }, + "why": "A NULL row does not embed to NULL, it stops the batch. Msg 8116, argument data type NULL is invalid, is this skill's measurement of 2026-08-28, and it is raised on the argument before anything leaves the engine, which is why this probe needs no reachable endpoint. It is the reason the batching statement in the skill filters on embedding IS NULL rather than trusting the function to pass nulls through, and an empty string by contrast is accepted and returns a real vector." + }, + { + "sql": "CREATE TABLE dbo.probe_em_large (id int NOT NULL PRIMARY KEY, embedding vector(3072) NOT NULL);", + "expect": { + "error": 2717 + }, + "why": "Which embedding model can be chosen at all, settled before a single call is made. Microsoft Learn's vector data type page states the maximum is 1998 dimensions, and its Azure OpenAI model table gives text-embedding-3-large an output of 3072, so the highest scoring model does not fit its default output into any column this engine can declare. The fix is PARAMETERS '{\"dimensions\":1536}' on the external model. Msg 2717 is the merged vector-search-azure-sql skill's measurement of the same refusal at 1999 dimensions, run against a real engine on 2026-09-03; that skill owns the column and this probe asserts only the consequence for model choice." + } + , + { + "sql": "EXEC sp_configure 'show advanced options', 1;", + "expect": { + "error": 2812 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The remedy this skill names for Msg 31644 cannot be typed where a reader is connected. sp_configure is not callable from a user database at all, which is the first thing a reader hits and is NOT the number this skill printed until 2026-09-06. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE EXTERNAL MODEL probe_em_apiformat WITH (LOCATION = N'https://example.com/x', API_FORMAT = N'NotAFormat', MODEL_TYPE = EMBEDDINGS, MODEL = N'text-embedding-3-small');", + "expect": { + "error": 46508 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. API_FORMAT takes a listed value and an unlisted one is refused at DDL time rather than at call time, so a typo here never reaches an endpoint. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "EXEC sp_invoke_external_rest_endpoint @url = N'https://10.0.0.5/x', @method = 'GET';", + "expect": { + "error": 31624 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A private address is closed off, which is half of why a locally hosted model cannot be reached from the container. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##') CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'Str0ng!Lab_Passw0rd_2026';\nGO\nCREATE DATABASE SCOPED CREDENTIAL [https://probe.example.com] WITH IDENTITY = 'HTTPEndpointHeaders', SECRET = '{\"Authorization\":\"Bearer k\"}';\nGO\nCREATE EXTERNAL MODEL probe_em_cred WITH (LOCATION = N'https://probe.example.com/x', API_FORMAT = N'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = N'text-embedding-3-small', CREDENTIAL = [https://probe.example.com]);\nGO\nDROP DATABASE SCOPED CREDENTIAL [https://probe.example.com];", + "expect": { + "error": 46556 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A credential a model still uses cannot be dropped, so a teardown script has to drop the model first. The whole fixture is built here rather than assumed, so the probe is about the ordering rule and not about leftover state. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE USER probe_em_low WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_em_low';\nEXEC sp_invoke_external_rest_endpoint @url = N'https://example.com/';\nREVERT;", + "expect": { + "error": 8189 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. EXECUTE ANY EXTERNAL ENDPOINT is a separate grant, so a user who can read the table still cannot call out. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = '##MS_DatabaseMasterKey##') CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'Str0ng!Lab_Passw0rd_2026';\nGO\nCREATE EXTERNAL MODEL probe_em_perm WITH (LOCATION = N'https://example.com/x', API_FORMAT = N'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = N'text-embedding-3-small');\nGO\nCREATE USER probe_em_exec WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_em_exec';\nDECLARE @v vector(1536) = AI_GENERATE_EMBEDDINGS(N'probe' USE MODEL probe_em_perm);\nREVERT;", + "expect": { + "error": 15151 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Without EXECUTE ON EXTERNAL MODEL the model appears not to exist, which is the permission failure that reads as a missing object and sends a developer looking in the wrong place. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE USER probe_em_ddl WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_em_ddl';\nCREATE EXTERNAL MODEL probe_em_denied WITH (LOCATION = N'https://example.com/x', API_FORMAT = N'Azure OpenAI', MODEL_TYPE = EMBEDDINGS, MODEL = N'text-embedding-3-small');\nREVERT;", + "expect": { + "error": 262 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Creating a model needs CREATE EXTERNAL MODEL, which db_datawriter does not carry. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "EXEC sp_configure 'show advanced options', 1;\nRECONFIGURE;", + "expect": { + "error": 40510 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. THE CORRECTION. Connected to a user database the previous probe shows sp_configure is not there; connected here it runs and stages the value without complaint. RECONFIGURE is the statement that refuses, so the staged value never takes effect and nothing says so. This skill attributed Msg 40510 to sp_configure until 2026-09-06, which sends a reader looking at the wrong statement. This probe runs against the scratch database the lane creates, so if sp_configure ever becomes callable there this goes red on the wrong number rather than passing quietly. It goes red if the statement succeeds, or fails with any other number." + } + , + { + "sql": "SELECT * FROM sys.dm_external_model_stats;", + "expect": { + "error": 208 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The parallelism view this skill tells a reader to size against does not exist on the container, so a plan that reads it locally and expects the cloud's answer is reading nothing. The skill's rule is to size parallelism in the cloud or not at all. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_em_dim (id int, v vector(3) NOT NULL);\nGO\nINSERT INTO dbo.probe_em_dim VALUES (1, N'[1,2,3,4]');", + "expect": { + "error": 42204 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The one loud failure in this surface, asserted through the engine rule it rests on rather than through a live model call: a value whose length does not match the declared dimension is refused at write time. What this probe does NOT establish is that AI_GENERATE_EMBEDDINGS produces the mismatch, which needs a real endpoint. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Use the embeddings-and-external-models skill", + "Create an external model and call the embedding function from T-SQL", + "Create an external model and call AI_GENERATE_EMBEDDINGS", + "How do I generate embeddings inside Azure SQL Database without an application", + "Which permissions does an external model endpoint need and how does the allowlist work", + "Call a REST endpoint from Azure SQL Database with sp_invoke_external_rest_endpoint", + "Chunk text in the database with AI_GENERATE_CHUNKS" + ], + "implicit": [ + "the database scoped credential cannot be used to invoke an external rest endpoint", + "my embedding call says the external model does not exist but I can see it in the catalog", + "connections to the domain are not allowed when my query calls an endpoint", + "fail to obtain or decrypt secret for credential when I use managed identity from SQL", + "accessing the external endpoint is only allowed via HTTPS", + "my update that embeds a whole table has been running for hours", + "can I create a chat model in Azure SQL Database and call it from a query", + "how do I authenticate the database to Azure OpenAI without putting a key in the connection string", + "which chunk types does the built in chunking function support" + ], + "routing": [ + "embed a column of text without writing an application loop", + "let the database call our embedding endpoint directly", + "set up the credential and permissions so a query can reach an external endpoint" + ], + "negative": [ + "What are the restrictions on the vector data type in Azure SQL Database", + "Why does my similarity query scan the whole table instead of using the vector index", + "Show me the syntax for VECTOR_DISTANCE and the distance metrics", + "Build a retrieval augmented generation pipeline over my document library on Azure SQL Database", + "Chunk my documents and keep the embeddings up to date as the source changes", + "Run a local RAG prototype offline against the Azure SQL Database container", + "Build a RAG scenario on the container", + "Set up full text search and rank the results", + "Design the schema for my orders and customers tables", + "Add retry and connection pooling to my data access layer" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth b/plugins/microsoft-sql-vscode/skills/entra-id-auth deleted file mode 120000 index e45999d..0000000 --- a/plugins/microsoft-sql-vscode/skills/entra-id-auth +++ /dev/null @@ -1 +0,0 @@ -../../../skills/entra-id-auth \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth/SKILL.md b/plugins/microsoft-sql-vscode/skills/entra-id-auth/SKILL.md new file mode 100644 index 0000000..e633a8a --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth/SKILL.md @@ -0,0 +1,382 @@ +--- +name: entra-id-auth +description: >- + Takes an application identity to a passwordless connection to Azure SQL Database, and diagnoses + it when that fails: sets the Microsoft Entra administrator, creates the database user for a + managed identity or service principal, and grants roles. Use for "set up Microsoft Entra + authentication for Azure SQL", "connect with a managed identity", "stop putting the database + password in configuration", or "turn on Microsoft Entra-only authentication", and for the + container once its MSSQL_AAD_ variables are set. Owns failures after a credential was + evaluated: Msg 33134 "Principal could not be resolved", Msg 33131 "duplicate display name", + 18456 "Login failed for user", 4060 "Cannot open database". What fails before that is + diagnose-connection-errors; the connection code itself is connect-from-dotnet, + connect-from-python or connect-from-typescript-and-node. +--- + +# Microsoft Entra ID authentication for an application identity + +Passwordless connection is an ordered procedure whose failures do not arrive in order: steps succeed +while achieving nothing, and the errors name the wrong component. + +Every flag below was read from the tool on 2026-09-03: Azure CLI 2.90.0, `sqlcmd` 1.10.0, `azd` +1.32.0, and the container behaviour was measured the same day. Cloud engine behaviour and message +text are from Microsoft Learn, cited in `references/`. + +## What this skill owns, and what it does not + +**Owns** the server-side identity configuration, the database user, the grant, the authentication +keyword, and every failure about **who the caller is**. Route the rest. + +| Question | Skill | +|---|---| +| Retry, encryption, pooling, and whether the container has Entra ID configured at all | `connect-to-azure-sql` | +| Drivers and per-language syntax | `connect-from-dotnet`, `connect-from-python`, `connect-from-typescript-and-node` | +| The server, the database, the firewall rule | `provision-azure-sql-db` | +| Transport, pre-login, certificates, timeouts, the `40xxx` gateway family | `diagnose-connection-errors` | +| Which roles the application should hold | `least-privilege-database-roles` | +| Server-level Entra logins and fixed server roles | `entra-logins-and-server-roles` | +| Identity through a hosting service, and the deployment | `managed-identity-across-azure-services`, `deploy-app-to-azure` | + +**The split, stated once.** `diagnose-connection-errors` owns failures before a credential is +evaluated; this skill owns those after it: `18456`, `4060`, `33134`, `33131`, `37545`. A failure +carrying no SQL error number is neither. + +## 18456 and 4060 arrive after the credential + +Both read as "login failed", neither is a network problem, and no edit to the server name, +encryption or timeout fixes either one. + +- **`18456`** means the credential was evaluated and refused. Naming + `` it means the opposite of what it looks like: the token was + **accepted**, and no matching principal exists in the database. +- **`4060`** means the login is valid and has no user in **that** database. After a deployment that + is almost always a missing database user: a login lives on the server, a user in the database, and + deployment tooling creates neither. + +Both are answered by step 5, run on the user database. `40532` reads the same and is the gateway +refusing before a database was reached, so it is `diagnose-connection-errors`. + +## The order that works + +Steps 2, 5 and 7 are the ones that fail quietly. + +1. **Set the Microsoft Entra administrator on the logical server.** Nothing works until it exists, + and the CLI takes an object id rather than a name, so look it up first. + + ```bash + az ad user list --display-name "" --query "[].{id:id, upn:userPrincipalName}" -o table + az sql server ad-admin create -g -s \ + --display-name "" --object-id + ``` + +2. **Decide whether the logical server needs its own identity.** It does **only** if a service + principal, not a signed-in person, runs `CREATE USER`, the normal case for a pipeline. See + [Msg 33134](#msg-33134-does-not-mean-what-it-says). + +3. **Give the application an identity, and take its client id from the deployment, not the portal.** + `azd` prints the client id among the environment values only if the infrastructure declared it + as an output, so add that output rather than copying a GUID. + + ```bash + azd env get-values -e + az ad sp show --id --query "{appId:appId, objectId:id}" + ``` + + `appId` is the client id step 5 wants. `objectId` is the enterprise application object id, which + only `WITH OBJECT_ID` wants. They are not interchangeable. + +4. **Connect to the user database with a Microsoft Entra authenticated connection.** Not `master`, + and not a SQL login. + + ```bash + sqlcmd -S .database.windows.net -d -U -G -l 30 + ``` + +5. **Create the database user.** `CREATE USER` adds the user to the **current** database, so the + database in step 4 is the one that gets it. Run against `master` it is valid, succeeds, and + reports nothing; the application then fails against the user database with correct code. + + ```sql + CREATE USER [] FROM EXTERNAL PROVIDER; + ``` + +6. **Grant roles, and name them.** Add `db_ddladmin` only if the application applies schema. A grant + made after the application first connected does not take until the host's token cache turns over, + so the grant is correct and the application keeps failing. + + ```sql + ALTER ROLE db_datareader ADD MEMBER []; + ALTER ROLE db_datawriter ADD MEMBER []; + ``` + +7. **Write the connection string**, from + [the per-driver table](#the-authentication-keyword-is-not-the-same-string-twice). + +8. **Check it worked from the database**, below. + +## Msg 33134 does not mean what it says + +```output +Msg 33134, Level 16, State 1, Line 1 +Principal 'test-user' could not be resolved. +Error message: 'Server identity is not configured. ...' +``` + +The first line sends an agent to check the principal name, which is almost always fine. **The fault +is on the logical server.** A **Microsoft Entra user** running `CREATE USER` is impersonated and +Graph queried with that user's permissions; an application cannot impersonate another application, +so a **service principal** falls back to the **server identity**. With no server identity, or one +that cannot read Graph, the lookup fails and the message blames the principal, which is why the same +statement works by hand and fails in a pipeline. + +```bash +az sql server update -g -n -i +az sql server show -g -n --query identity +``` + +**The flag is `-i`, long form `--assign_identity` with an underscore.** Measured 2026-09-03 on Azure +CLI 2.90.0: the hyphenated `--assign-identity` exits with `unrecognized arguments`, so a script +carrying it assigns nothing and the next `CREATE USER` still raises `33134`. + +Then, in order of tenant privilege needed: grant that identity Microsoft Graph read, either three +application permissions or the broader `Directory Readers` role, both needing a `Privileged Role +Administrator`; or put it in a role-assignable group holding `Directory Readers`, the production +shape, because a group owner can then add servers; or skip the lookup with the third `CREATE USER` +form below, the answer wherever that privilege is absent. Open +[references/identity-errors.md](references/identity-errors.md) before attempting any of the three, +because it names the exact Graph permissions. + +## Three forms of CREATE USER, and when each is forced + +| Form | Looks the principal up | Use it when | +|---|---|---| +| `CREATE USER [] FROM EXTERNAL PROVIDER;` | Yes | The default. A person is running it, or the server identity is configured | +| `CREATE USER [] FROM EXTERNAL PROVIDER WITH OBJECT_ID = '';` | Yes | The display name is **not unique** in the tenant, so the plain form fails | +| `CREATE USER [] WITH SID = , TYPE = E;` | **No** | No Graph permission is available, or Msg 33134 has to be worked around. Needs an engine that has Microsoft Entra configured, so on a bare container it raises `40530` | + +Agents paraphrase the third form into something that does not run, because the id must be converted +to binary. Open [references/identity-errors.md](references/identity-errors.md) and copy the worked +block there: `TYPE = E` covers users, applications and managed identities, `TYPE = X` groups. + +**Nothing validates that id**, so a wrong value creates a user no token will ever match and it reads +back as a healthy row. For a **user or group** it is the **object id**; for a **service principal or +managed identity** the **application (client) id**. + +**`WITH OBJECT_ID` is not the general fix and does not answer Msg 33134.** It exists for one problem: +Microsoft Entra ID permits duplicate application display names and the engine requires a unique one. +That produces a different error. + +```output +Msg 33131, Level 16, State 1, Line 4 +Principal 'myapp' has a duplicate display name. +``` + +Three traps. **`WITH OBJECT_ID` is a modifier on `FROM EXTERNAL PROVIDER`, not an alternative to +it**: on its own the statement is `Msg 37546, Can only specify object_id when creating user from +external provider`, measured 2026-09-08. The object id must exist in this tenant or the statement +fails with `Msg 37545`. And the **Object ID on an app registration is not the one on its service +principal**: this clause wants the enterprise application one, `objectId` in step 3. + +## A SQL-authenticated administrator cannot do any of this + +Learn is explicit: only Microsoft Entra users can create other Entra users in Azure SQL Database, +and no SQL-authenticated user can, the server administrator included, which is why "I am the admin +and it still fails" is a dead end. The permission needed is `ALTER ANY USER`, carried by `db_owner`, +but no grant substitutes for the connection being Entra authenticated. The `WITH SID` form is the +exception: it performs no external lookup. + +## The name to put in the brackets + +A wrong name produces **`Msg 33130`**, `Principal '' could not be found or this principal type +is not supported`, and reads as a permissions problem. Measured on two different logical servers, on +2026-09-05 and again on 2026-09-08: an unknown address, a bare guid and a non-existent application +name all answered 33130. `Msg 33134` is a different path, above; `Msg 33131` is the duplicate +display name, confirmed 2026-09-08 by `CREATE USER [Microsoft Graph] FROM EXTERNAL PROVIDER`. The one +nobody guesses: a **system-assigned** identity on a deployment slot is `/slots/`, +which is why an application works in production and fails in staging. Open +[references/identity-errors.md](references/identity-errors.md) before typing a name you inferred, +because it tables all five principal kinds. + +## The authentication keyword is not the same string twice + +Every value below is plausible in every driver and each is valid in only one of them. + +| Stack | Passwordless default | Managed identity, explicit | User-assigned client id goes in | +|---|---|---|---| +| .NET, `Microsoft.Data.SqlClient` | `Authentication=Active Directory Default` | `Active Directory Managed Identity`, alias `Active Directory MSI` | `User Id=` | +| Python, `mssql-python` | `Authentication=ActiveDirectoryDefault` | `Authentication=ActiveDirectoryMSI` | `UID=` | +| Python, `pyodbc` on ODBC Driver 18 | **no such value exists** | `Authentication=ActiveDirectoryMsi` | `UID=`, object id on most hosts | +| Node.js, `mssql` on `tedious` | `type: 'azure-active-directory-default'` | `azure-active-directory-msi-app-service`, `azure-active-directory-msi-vm` | `authentication.options.clientId` | +| Java, the Microsoft JDBC driver | `authentication=ActiveDirectoryDefault` | `ActiveDirectoryManagedIdentity`, alias `ActiveDirectoryMSI` | `msiClientId`, or `user` | + +**In production, name the managed identity mode, not the default.** The default walks a chain of +credential providers, which Microsoft warns costs response time, and can pick up a developer +credential on a machine that has one. + +Two facts a plausible guess gets wrong. **ODBC has no `ActiveDirectoryDefault`** and spells managed +identity `ActiveDirectoryMsi`, mixed case, so Python is two stories: `mssql-python` has a default +mode and uses `ActiveDirectoryMSI`, `pyodbc` has neither. **In Node.js the client id is nested**, in +`authentication.options.clientId`, not the connection `options` bag beside `encrypt`. + +Open [references/driver-auth-keywords.md](references/driver-auth-keywords.md) before writing one of +these values, because each list there is the driver's complete accepted set and a value outside it +is not a value. A driver that names its own set, such as `sqlcmd`, is faster to ask than to recall. + +**`Microsoft.Data.SqlClient` 7.0 needs the `Microsoft.Data.SqlClient.Extensions.Azure` package**, or +every `Active Directory *` mode fails at run time; connection strings do not change. `Active +Directory Password` is deprecated across the drivers and `[Obsolete]` in 7.0, because it rests on +the resource owner password credentials grant. + +```bash +dotnet add package Microsoft.Data.SqlClient.Extensions.Azure +``` + +## System-assigned or user-assigned + +Two decisions, confused because both use the same words. **The server identity** reads Microsoft +Graph and matters only for step 2; a user-assigned one survives the server and holds the Graph +permissions once for many. **The application identity** is what the application authenticates as, +and changes the connection string **only if user-assigned**, which must name its client id because +the host may carry several. + +## On the Azure SQL Database container, the shape is login then user + +The container reads a certificate, the application (client) id and the tenant id from three +environment variables set at creation. One command says which case you are in. + +```bash +docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' | grep -c '^MSSQL_AAD_' +``` + +`0` means Entra ID was never configured: an Entra login returns `18456`, `Login failed for user ''`, +which is `connect-to-azure-sql`, not this skill, and the error below. `3` means `MSSQL_AAD_CLIENT_ID`, +`MSSQL_AAD_PRIMARY_TENANT` and `MSSQL_AAD_CERTIFICATE_FILE_PATH` are set and the database side is +yours. Learn documents that trio, and the statements below, for the SQL Server on Linux image and not +for this one, so confirm the engine agreed rather than assuming: + +```bash +docker exec bash -c "grep Entra /var/opt/mssql/log/errorlog" +``` + +Expected: `Microsoft Entra ID authentication is enabled.` The principal is then a server login and +a database user from that login, not a contained user: + +```sql +CREATE LOGIN [] FROM EXTERNAL PROVIDER; +CREATE USER [] FROM LOGIN []; +``` + +### Msg 40530 blames the batch, and the batch is fine + +In the `0` case the no-lookup form is refused too, by a message naming the wrong thing. Measured +2026-09-03 with no `MSSQL_AAD_` variables: + +```output +Msg 40530, Level 16, State 1, Line 1 +The CREATE USER statement must be the only statement in the batch. +``` + +It **was** the only statement in the batch. `TYPE = X` fails identically, and so does every rewrite +tried, `EXEC (@cmd)` and `GO` separation included, so a reader who trusts the text rewrites the batch +forever and never succeeds. `CREATE USER [] WITHOUT LOGIN;`, the same batch shape, succeeds, +which rules the batch out. **The fix is server configuration, not SQL**: the engine has no Microsoft +Entra configuration and refuses the external principal types. Set the three variables and recreate +the container, or run against Azure SQL Database. Open +[references/identity-errors.md](references/identity-errors.md) before believing the number, because +Microsoft Learn documents `40530` only as the batch rule and that disagreement is recorded there. + +Two things have no local equivalent, so a green local run proves nothing about either: the **server +identity** and its Graph permission, which `Msg 33134` is about, and **Entra-only authentication** +with its Azure Policy enforcement. + +## Microsoft Entra-only authentication + +Enabling it disables SQL authentication at the **server** level, for every database and the server +administrator too. Existing SQL logins are not removed and new ones can still be created; they +simply cannot connect. + +```bash +az sql server ad-only-auth enable -g -n +az sql server ad-only-auth get -g -n +``` + +**The administrator must be set first**, or enabling fails through every interface, and it cannot be +removed until the setting is disabled again. Enabling also silently disables elastic jobs, SQL Data +Sync, SQL Insights, `EXEC AS` for Entra group members and some change data capture paths. + +**Enforcing it takes two Azure Policy definitions, not one**: one evaluates a server at creation, +the other the setting on an existing server. Open +[references/entra-only-and-policy.md](references/entra-only-and-policy.md) before enabling this on a +shared server or naming a policy in a plan, because it carries the full disabled-feature list, the +role split, and the two definition ids. + +## Check it worked + +Run the first two against the **user database**, never `master`. + +**1. The principal exists and its stored id is the identity you meant.** + +```sql +SELECT name, type, type_desc, + CAST(CAST(sid AS varbinary(16)) AS uniqueidentifier) AS entra_id, + DATALENGTH(sid) AS sid_bytes +FROM sys.database_principals +WHERE type IN ('E', 'X'); +``` + +Expected: `type_desc` is `EXTERNAL_USER` for an application or a person, `EXTERNAL_GROUP` for a +group. `entra_id` must equal the client id the workload runs as; a row carrying the wrong one looks +exactly like success. `sid_bytes` is `16` for a contained user, `18` for one from a server login. On +a container with no `MSSQL_AAD_` variables it returns nothing: no `E` or `X` principal exists there. + +**2. The roles are the ones you named, and nothing wider.** + +```sql +SELECT r.name AS role_name, m.name AS member_name +FROM sys.database_role_members rm +JOIN sys.database_principals r ON r.principal_id = rm.role_principal_id +JOIN sys.database_principals m ON m.principal_id = rm.member_principal_id +WHERE m.type IN ('E', 'X'); +``` + +Expected: exactly the roles from step 6. A `db_owner` or `CONTROL` grant nobody asked for is the +usual leftover of a one-command tool. + +**3. On the cloud, the server side agrees.** + +```bash +az sql server ad-admin list -g -s -o table +az sql server show -g -n --query identity +``` + +Expected: the administrator from step 1, and a non-null identity if step 2 said you needed one. +Having run the command is not evidence it took. + +## References + +Each of the three files under `references/` is linked from the section that needs it, at the point +where the condition for opening it is stated. Tenant-side role assignment changes, so fetch the Learn +articles rather than recalling them. + +## Do not + +- Do not answer `Msg 33134` by editing the principal name. Check the server identity, its Graph + permission, and that the assign command used `-i`. +- Do not reach for `WITH OBJECT_ID` as a general fix. It answers `Msg 33131` and nothing else, and + wants the enterprise application object id, not the app registration's. +- Do not run `CREATE USER FROM EXTERNAL PROVIDER` on `master` and report the identity as configured. +- Do not attempt the lookup forms from a SQL-authenticated connection, server administrator included. + No grant makes those work. +- Do not transpose an authentication keyword between drivers, and never write + `ActiveDirectoryDefault` into an ODBC connection string. +- Do not use `Active Directory Password` in any spelling, or leave a .NET 7.0 application on an + `Active Directory *` mode without the extension package. +- Do not put a client secret in a connection string on Azure, or grant `CONTROL ON DATABASE` because + a tool did. Role design is `least-privilege-database-roles`. +- Do not enable Entra-only authentication on a shared server without naming what it turns off, or + assign one Azure Policy definition and call the estate enforced. +- Do not conclude a grant failed until the identity token cache has turned over. +- Do not answer `Msg 40530` by rewriting the batch. On a bare container it means the engine has no + Entra configuration. +- Do not read a clean container run as evidence the cloud path works. The server identity, `Msg + 33134` and Entra-only authentication have no local equivalent. diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/driver-auth-keywords.md b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/driver-auth-keywords.md new file mode 100644 index 0000000..4daf4f8 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/driver-auth-keywords.md @@ -0,0 +1,214 @@ +# The authentication keyword, per driver, verbatim + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [Microsoft.Data.SqlClient, for .NET](#microsoftdatasqlclient-for-net) +- [ODBC Driver 18 for SQL Server, including pyodbc](#odbc-driver-18-for-sql-server-including-pyodbc) +- [mssql-python](#mssql-python) +- [tedious, under the Node.js mssql package](#tedious-under-the-nodejs-mssql-package) +- [The Microsoft JDBC driver](#the-microsoft-jdbc-driver) +- [sqlcmd](#sqlcmd) +- [The same idea, five spellings](#the-same-idea-five-spellings) +- [Password authentication is deprecated everywhere](#password-authentication-is-deprecated-everywhere) +- [Sources](#sources) + +## How to use this file + +Read it before writing an authentication keyword into a connection string. Each list below is the +**complete** documented set for that driver, so a value not in the list is not a value. + +The failure this prevents is quiet: a driver that does not recognise the value may fall through to +a different mechanism and produce a login failure that reads as a database permission problem. + +## Microsoft.Data.SqlClient, for .NET + +The only family that writes the values **with spaces**. + +| Value | Since | +|---|---| +| `Active Directory Integrated` | 2.0.0 | +| `Active Directory Interactive` | 2.0.0 | +| `Active Directory Service Principal` | 2.0.0 | +| `Active Directory Device Code Flow` | 2.1.0 | +| `Active Directory Managed Identity`, and the alias `Active Directory MSI` | 2.1.0 | +| `Active Directory Default` | 3.0.0 | +| `Active Directory Workload Identity` | 5.2.0 | +| `Active Directory Password` | 1.0, deprecated | + +- User-assigned managed identity: `User Id=`. On 3.0 and later that is the **client id**; + on 2.1 it was the object id. +- `Active Directory Default` reads the same `User Id` property for the managed identity client id. +- `Active Directory Workload Identity` also takes the client id in `User Id`, and otherwise defaults + from environment variables. +- Microsoft warns that `Active Directory Default` "can come with performance impacts because it has + to look in multiple places for authentication information", and is not recommended where response + times are tight. Name the mode explicitly in production. + +**Version 7.0 is a breaking change.** The core package no longer carries the Azure and Entra +dependencies, and any of these modes needs the `Microsoft.Data.SqlClient.Extensions.Azure` package +added. Connection strings do not change. `SqlAuthenticationMethod.ActiveDirectoryPassword` became +`[Obsolete]` in the same release. An application that authenticated fine before the upgrade and +fails after it is usually missing that package reference, and says so at run time with +`Cannot find an authentication provider for 'ActiveDirectoryDefault'.` + +```bash +dotnet list package --include-transitive | grep Microsoft.Data.SqlClient +``` + +## ODBC Driver 18 for SQL Server, including pyodbc + +The complete documented set, verbatim from the keyword table: + +> `(not set)`(default), `(empty string)`, `SqlPassword`, `ActiveDirectoryIntegrated`, +> `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, `ActiveDirectoryServicePrincipal`, +> `ActiveDirectoryPassword` [DEPRECATED] + +**There is no `ActiveDirectoryDefault` and no `ActiveDirectoryManagedIdentity` in ODBC.** There is +also no device code mode. Writing any of those into a `pyodbc` connection string is the single most +common invented value in this area. + +Managed identity, quoted: + +> `ActiveDirectoryMsi` - Authenticate with a Microsoft Entra managed identity. For a user-assigned +> identity, set UID to the identity's client ID for Azure App Service or Azure Container Instance; +> otherwise, use its object ID. For system-assigned identity, UID isn't required. + +```text +server=;database=;Authentication=ActiveDirectoryMsi;Encrypt=yes; +server=;database=;UID=;Authentication=ActiveDirectoryMsi;Encrypt=yes; +``` + +Version notes: managed identity arrived in 17.3.1.1 for both identity kinds; +`ActiveDirectoryServicePrincipal` in 17.7, where `UID` is the client id and `PWD` the client secret. + +A token can be supplied instead through the `SQL_COPT_SS_ACCESS_TOKEN` pre-connection attribute. +When it is used, the connection string must not contain `UID`, `PWD`, `Authentication` or +`Trusted_Connection`. + +## mssql-python + +Seven modes, all through the `Authentication` connection-string keyword. + +| Value | What it does | +|---|---| +| `ActiveDirectoryDefault` | `DefaultAzureCredential`, which walks a chain of providers | +| `ActiveDirectoryInteractive` | Browser sign-in | +| `ActiveDirectoryDeviceCode` | Code entry on a second device | +| `ActiveDirectoryMSI` | Managed identity, either kind | +| `ActiveDirectoryServicePrincipal` | `UID` is the client id, `PWD` the client secret | +| `ActiveDirectoryIntegrated` | Kerberos, in a federated domain | +| `ActiveDirectoryPassword` | Deprecated | + +`ActiveDirectoryDefault`, `ActiveDirectoryInteractive` and `ActiveDirectoryDeviceCode` need the +`azure-identity` package installed alongside the driver. + +User-assigned managed identity, quoted: "Specify the client ID of a user-assigned managed identity +in the `UID` field." + +```python +"Server=.database.windows.net;" +"Database=;" +"Authentication=ActiveDirectoryMSI;" +"UID=;" +"Encrypt=yes;" +``` + +Note the spelling: `ActiveDirectoryMSI` here, `ActiveDirectoryMsi` in ODBC, and +`ActiveDirectoryManagedIdentity` in neither. + +## tedious, under the Node.js mssql package + +The value is an object, not a string, and the driver validates the type against a fixed list: +`default`, `ntlm`, `token-credential`, `azure-active-directory-password`, +`azure-active-directory-access-token`, `azure-active-directory-default`, +`azure-active-directory-msi-vm`, `azure-active-directory-msi-app-service`, +`azure-active-directory-service-principal-secret`. + +```js +authentication: { + type: 'azure-active-directory-default', + options: { clientId: '' } +} +``` + +**The client id is read from `authentication.options.clientId` and nowhere else.** A sample that +places `clientId` in the connection `options` bag beside `encrypt` appears to work only when the +matching environment variable is also set, which the credential chain picks up on its own. Remove +the environment variable and the same code silently falls back to the system-assigned identity. + +`token-credential` takes a credential object from the identity library directly, and is the +cleanest option when the credential is already constructed in application code. + +The Node.js package's own README lists an older, shorter set of types. It passes `authentication` +straight through, so the driver's list above is the authority. + +## The Microsoft JDBC driver + +`authentication=` accepts `NotSpecified` (the default), `SqlPassword`, `ActiveDirectoryPassword`, +`ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, `ActiveDirectoryServicePrincipal`, +`ActiveDirectoryServicePrincipalCertificate`, `ActiveDirectoryManagedIdentity` and +`ActiveDirectoryDefault`. + +- `ActiveDirectoryMSI` is accepted as a **synonym** for `ActiveDirectoryManagedIdentity`. Prefer the + long form on driver 12.2 and later. +- User-assigned client id goes in `msiClientId`, and from 12.2 the `user` property is also accepted. + JDBC takes the **client** id, unlike ODBC. +- Dependencies split by mode: the MSAL library for the interactive, integrated, password and service + principal modes, and the Azure identity library for the managed identity and default modes. + Missing the right one produces a class-loading failure rather than a login failure. + +## sqlcmd + +Verified by running the tool at version 1.10.0. Supplying an invalid value prints the accepted set: + +```text +Invalid federated authentication type 'NotARealMode': expected one of [ActiveDirectoryApplication +ActiveDirectoryServicePrincipal ActiveDirectoryDefault ActiveDirectoryIntegrated +ActiveDirectoryInteractive ActiveDirectoryManagedIdentity ActiveDirectoryMSI ActiveDirectoryPassword +ActiveDirectoryAzCli ActiveDirectoryDeviceCode ActiveDirectoryAzureDeveloperCli +ActiveDirectoryAzurePipelines ActiveDirectoryEnvironment ActiveDirectoryWorkloadIdentity +ActiveDirectoryClientAssertion ActiveDirectoryOnBehalfOf] +``` + +`-G` selects Entra authentication with the interactive or default flow, and +`--authentication-method=` names one of the above. Asking the tool is faster and more +reliable than recalling the list, and it is a good habit for any driver that will tell you. + +## The same idea, five spellings + +One concept, managed identity, across the drivers: + +| Driver | The value | +|---|---| +| `Microsoft.Data.SqlClient` | `Active Directory Managed Identity` | +| ODBC Driver 18 | `ActiveDirectoryMsi` | +| `mssql-python` | `ActiveDirectoryMSI` | +| `tedious` | `azure-active-directory-msi-app-service` or `azure-active-directory-msi-vm` | +| JDBC | `ActiveDirectoryManagedIdentity` | + +And the passwordless default mode does not exist at all in ODBC. + +## Password authentication is deprecated everywhere + +The drivers carry the same wording: + +> The ActiveDirectoryPassword authentication option (Microsoft Entra ID password authentication) is +> deprecated in the Microsoft SQL drivers. This high-risk authentication flow is incompatible with +> mandatory Microsoft Entra multifactor authentication (MFA) and might not work in tenants where MFA +> is enforced. Plan to migrate to a different Microsoft Entra authentication method. + +It rests on the resource owner password credentials grant. No removal version is stated anywhere, so +it is a deprecation rather than a removal, but a tenant enforcing multifactor authentication already +breaks it. The documented replacements: an interactive or integrated mode when a person is present, +managed identity on Azure, and a service principal with a certificate rather than a secret elsewhere. + +## Sources + +All read on 2026-08-27. Fetch rather than trust this summary when a value is load-bearing. + +- SqlClient: `/sql/connect/ado-net/sql/azure-active-directory-authentication`, which carries the value table, the version column and the 7.0 migration section. +- ODBC: `/sql/connect/odbc/using-azure-active-directory`, which carries the keyword table and the managed identity id rule. +- mssql-python: `/sql/connect/python/mssql-python/entra-authentication`, which carries the seven modes and the `UID` rule. +- JDBC: `/sql/connect/jdbc/connecting-using-azure-active-directory-authentication`, which carries the property values, the synonym and the dependency table. +- tedious: the driver's own source and released documentation, which is the authority over the wrapping package's README. diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/entra-only-and-policy.md b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/entra-only-and-policy.md new file mode 100644 index 0000000..66f35bd --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/entra-only-and-policy.md @@ -0,0 +1,140 @@ +# Microsoft Entra-only authentication, and enforcing it + +## Contents + +- [What enabling it actually does](#what-enabling-it-actually-does) +- [Order, and who is allowed to do it](#order-and-who-is-allowed-to-do-it) +- [What stops working](#what-stops-working) +- [The two Azure Policy definitions](#the-two-azure-policy-definitions) +- [Reading the policies rather than trusting a summary](#reading-the-policies-rather-than-trusting-a-summary) +- [Sources](#sources) + +## What enabling it actually does + +> When enabling Microsoft Entra-only authentication, SQL authentication is disabled at the server +> level and prevents any authentication based on any SQL authentication credentials. + +Three consequences worth being explicit about: + +- It applies to the **whole logical server**, so every database on it, and to the server + administrator login as well. +- Existing SQL logins and users are **not removed**. New ones can still be created by an Entra + account with the right permission. They simply cannot connect. +- Entra users with the right permission can still impersonate SQL users, and impersonation between + SQL authentication users keeps working while the feature is on. + +```bash +az sql server ad-only-auth enable -g -n +az sql server ad-only-auth disable -g -n +az sql server ad-only-auth get -g -n +``` + +```sql +SELECT SERVERPROPERTY('IsExternalAuthenticationOnly'); -- 1 enabled, 0 disabled +``` + +The CLI subcommands still carry the older Active Directory naming, which is expected: the product +name changed and the identifiers deliberately did not. + +## Order, and who is allowed to do it + +- **The Microsoft Entra administrator must be set before enabling.** Without one the setting stays + inactive, and enabling through the APIs fails as well. +- **The administrator cannot be removed while it is enabled.** Removing it through an API fails. + Removing the administrator and disabling the setting together is allowed with the right + permissions. + +The Azure roles are split on purpose, and neither of the obvious SQL roles can do the whole job. + +| Role | Set the Entra administrator | Toggle Entra-only | +|---|---|---| +| `SQL Server Contributor` | Yes | No | +| `SQL Security Manager` | No | Yes | +| `Contributor`, `Owner` | Yes | Yes | + +The rationale is separation of duties: a role that can create a server or set an administrator +should not also be able to turn a security control off. The actions behind the setting are +`Microsoft.Sql/servers/azureADOnlyAuthentications/*`, plus +`Microsoft.Sql/servers/administrators/read` for anyone who needs to see it in the resource pages. + +## What stops working + +In Azure SQL Database, with Entra-only authentication on: + +- Elastic jobs +- SQL Data Sync +- SQL Insights +- `EXEC AS` for Microsoft Entra group member accounts +- Change data capture across the boundary: a database created by an Entra user has CDC artifacts a + SQL user cannot change, and the reverse holds too +- Transactional replication into the database, because that path requires SQL authentication between + participants + +Check this list against what already runs on the server. Nothing warns at enable time. + +## The two Azure Policy definitions + +Both are built in, both default to the `Audit` effect, and both accept `Audit`, `Deny` and +`Disabled`. Read on 2026-08-27 from the live definitions. + +| Display name | Definition id | Evaluates | +|---|---|---| +| `Azure SQL logical servers should have Microsoft Entra-only authentication enabled during creation` | `abda6d70-9778-44e7-84a8-06713e6db027` | `Microsoft.Sql/servers` at create time | +| `Azure SQL Database should have Microsoft Entra-only authentication enabled` | `b3a22bc9-66de-45fb-98fa-00f5df42f41a` | `Microsoft.Sql/servers/azureADOnlyAuthentications` on an existing server | + +Their own descriptions say why one is not enough: + +> Require Azure SQL logical servers to be created with Microsoft Entra-only authentication. This +> policy doesn't block local authentication from being re-enabled on resources after create. + +> Require Azure SQL logical servers to use Microsoft Entra-only authentication. This policy doesn't +> block servers from being created with local authentication enabled. It does block local +> authentication from being enabled on resources after create. + +Both then say: "Consider using the 'Microsoft Entra-only authentication' initiative instead to +require both." + +The initiative is **`Azure SQL Database should have Microsoft Entra-only authentication`**, +`a55e4a7e-1b9c-43ef-b4b3-642f303804d6`, and it contains exactly those two definitions. + +Two details the definitions carry that a summary loses: + +- The creation-time policy **exempts a server whose `createMode` is `Restore`**, so a restored + server can arrive non-compliant without the assignment complaining at create time. +- Both exclude servers whose resource group is managed by the analytics workspace provider. + +A server can be created already compliant, and the administrator set in the same call: + +```bash +az sql server create -g -n -l \ + --enable-ad-only-auth \ + --external-admin-principal-type User \ + --external-admin-name "" \ + --external-admin-sid +``` + +## Reading the policies rather than trusting a summary + +The published article about these policies is older than the definitions and still names them with +the previous product wording, and it describes two definitions where the live catalog carries a +create-time and a steady-state pair per product plus an initiative. Reading the live definitions is +one command and settles it: + +```bash +az policy definition list \ + --query "[?displayName!=null && contains(displayName, 'Entra-only authentication')].{display:displayName, id:name, allowed:parameters.effect.allowedValues, def:parameters.effect.defaultValue}" \ + -o json + +az policy set-definition list \ + --query "[?displayName!=null && contains(displayName, 'Entra-only')].{display:displayName, id:name}" \ + -o json +``` + +Do the same before quoting a policy name in a plan. The names in this file were read that way, not +recalled, and they should be re-read rather than trusted after a few months. + +## Sources + +- Microsoft Entra-only authentication with Azure SQL: `/azure/azure-sql/database/authentication-azure-ad-only-authentication`, which carries the feature description, the permission split and the unsupported feature list. +- Azure Policy for Microsoft Entra-only authentication: `/azure/azure-sql/database/authentication-azure-ad-only-authentication-policy`, which carries the effects and the compliance behaviour, and whose policy names are behind the live catalog. +- The live definitions, read with Azure CLI 2.89.1 on 2026-08-27. diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/identity-errors.md b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/identity-errors.md new file mode 100644 index 0000000..d95c0b5 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth/references/identity-errors.md @@ -0,0 +1,230 @@ +# Identity errors, and the three forms of CREATE USER + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [The error numbers, with their real causes](#the-error-numbers-with-their-real-causes) +- [Msg 33134 in full](#msg-33134-in-full) +- [Msg 33131 and the OBJECT_ID form](#msg-33131-and-the-object_id-form) +- [Msg 40530, where the message and the cause disagree](#msg-40530-where-the-message-and-the-cause-disagree) +- [The no-lookup form, with worked T-SQL](#the-no-lookup-form-with-worked-t-sql) +- [The name to put in the brackets](#the-name-to-put-in-the-brackets) +- [Which id goes where](#which-id-goes-where) +- [What the server identity is for](#what-the-server-identity-is-for) +- [Verification queries](#verification-queries) +- [Sources](#sources) + +## How to use this file + +Read it when an error number needs its exact text and its actual cause, or when the plain +`CREATE USER [] FROM EXTERNAL PROVIDER` has failed and the next form has to be chosen +deliberately. Sources are listed at the end. Numbers that are not about identity live in +`diagnose-connection-errors`. + +## The error numbers, with their real causes + +| Number | What the text says | What it usually is | +|---|---|---| +| `33134` | The principal could not be resolved | The logical server has no identity, or that identity cannot read Microsoft Graph. Only ever seen when a service principal runs the statement | +| `33131` | The principal has a duplicate display name | Microsoft Entra ID allows two applications to share a display name and the engine requires a unique one. Answer with `WITH OBJECT_ID` and an alias | +| `37545` | The object id is not valid, or the caller lacks permission | The object id given to `WITH OBJECT_ID` does not exist in this tenant, or the wrong one of the two portal object ids was used | +| `18456` | `Login failed for user ''.` | The credential was evaluated and refused: the login does not exist, is disabled, or the secret is wrong | +| `18456` naming `` | Login failed for that literal user name | The token was accepted and no matching principal exists in the database. The database user was never created, or was created in the wrong database | +| `40530` | The `CREATE USER` statement must be the only statement in the batch | On an engine with no Microsoft Entra configuration, the refusal of `TYPE = E` and `TYPE = X`. The batch is not the cause. See below | +| `4060` | `Cannot open database "" requested by the login. The login failed.` | The login is valid and has no user in that database, or the database name is wrong. After a deployment, almost always a missing database user | + +`18456` and `4060` are answered here, identity-shaped or plain, because both arrive after the +credential was evaluated. `40532` reads identically and is the gateway refusing before a database +was reached, so it belongs to `diagnose-connection-errors`. + +## Msg 33134 in full + +```output +Msg 33134, Level 16, State 1, Line 1 +Principal 'test-user' could not be resolved. +Error message: 'Server identity is not configured. Please follow the steps in "Assign an Azure AD +identity to your server and add Directory Reader permission to your identity" +(https://aka.ms/sqlaadsetup)' +``` + +The mechanism, quoted: + +> When a Microsoft Entra user executes these commands, Azure SQL's Microsoft application uses +> delegated permissions to impersonate the signed-in user and queries Microsoft Graph using their +> permissions. This flow isn't possible with service principals, because an application can't +> impersonate another application. Instead, the SQL engine tries to use its server identity ... The +> server identity must exist and have the Microsoft Graph query permissions or the operations fail. + +So the same statement succeeds when a person runs it and fails from a pipeline, which is the part +that makes it look like a permissions bug in the pipeline's own credential. + +Assign the identity with the commands in the skill body. **The flag is `-i`, long form +`--assign_identity` with an underscore.** Measured 2026-09-03 on Azure CLI 2.90.0: the hyphenated +`--assign-identity` exits with `unrecognized arguments`, so a script carrying it assigns nothing and +the next `CREATE USER` raises `33134` again. + +The Graph side needs a `Privileged Role Administrator` and can only be done from a script. Grant +either the three application permissions, the least-privilege option, or the `Directory Readers` +role, which is broader than the server needs: + +- `User.Read.All` +- `GroupMember.Read.All` +- `Application.Read.All` + +For production, Microsoft recommends a role-assignable group holding `Directory Readers`, so a +group owner can add server identities without a `Privileged Role Administrator` in the loop for +each one. + +## Msg 33131 and the OBJECT_ID form + +```output +Msg 33131, Level 16, State 1, Line 4 +Principal 'myapp' has a duplicate display name. Make the display name unique in Azure Active +Directory and execute this statement again. +``` + +The documented answer: + +```sql +CREATE USER [] FROM EXTERNAL PROVIDER + WITH OBJECT_ID = ''; +``` + +Four rules the documentation is explicit about: + +1. **Only for nonunique names.** Learn calls it a troubleshooting repair item for nonunique service + principals, and says the plain statement should be used otherwise. +2. **Add a suffix to the name.** Without one the statement succeeds and nothing records which + principal it was for. The recommended shape is the original name plus the first five characters + of the object id, for example `myapp2ba6c`, and the alias must fit `sysname`, at most 128 + characters. +3. **The object id must exist in this tenant**, or the statement fails with + `Msg 37545, Level 16, State 1, Line 1 '' is not a valid object id for '' or you do not have + permission.` +4. **It is the enterprise application object id**, not the one shown on the app registration page. + The documentation carries a warning about exactly this pair, because both are labelled Object ID + and only one works. + +Naming an Azure resource the same as an existing app registration is one way to arrive here: two +principals then share a display name, and `Msg 33131` is the collision being reported. + +The display name in Microsoft Entra ID and the alias in the database are not synchronised in either +direction. Renaming one never changes the other. + +## Msg 40530, where the message and the cause disagree + +```output +Msg 40530, Level 16, State 1, Line 1 +The CREATE USER statement must be the only statement in the batch. +``` + +Measured 2026-09-03 on an Azure SQL Database container started with no `MSSQL_AAD_` variables, `EngineEdition` `5`, +`IsExternalAuthenticationOnly` `0`: a lone `CREATE USER [p5] WITH SID = 0x1111..., TYPE = E;` raises +it, `TYPE = X` raises it, `EXEC (@cmd)` and `GO` separation raise it, and a lone +`CREATE USER [p4] WITHOUT LOGIN;` succeeds. The batch is not the cause. The engine has no Microsoft +Entra configuration, refuses the external principal types, and reports a batch rule. + +**Microsoft Learn documents `40530` only as the batch rule**: the errors and events table gives +severity 16 and the text `The %.*ls statement must be the only statement in the batch.`, with no +documented connection to Microsoft Entra configuration. The observed behaviour does not match the +documented meaning. That is recorded here as a disagreement rather than resolved: either the engine +reuses a number for a cause it has no message of its own for, or the documentation is incomplete, +and neither has been confirmed with the product team. + +## The no-lookup form, with worked T-SQL + +Azure SQL Database accepts a form of `CREATE USER` that does not query Microsoft Graph at all, so it +needs no server identity and no Graph permission. It is the practical answer to `Msg 33134` when +tenant-level privilege is not available. Nothing validates the id, so a wrong value produces a user +that no token will ever match. + +One block covers all three principals: change the id and the `TYPE`, per the table below. `TYPE` is +`E` for a user, an application or a managed identity, and `X` for a group. + +```sql +DECLARE @principal_name SYSNAME = 'example-app'; +DECLARE @id UNIQUEIDENTIFIER = ''; +DECLARE @castId NVARCHAR(MAX) = CONVERT(VARCHAR(MAX), CONVERT (VARBINARY(16), @id), 1); +DECLARE @cmd NVARCHAR(MAX) = N'CREATE USER [' + @principal_name + '] WITH SID = ' + @castId + ', TYPE = E;'; +EXEC (@cmd); +``` + +**This form is cloud, or an Entra-configured engine, only.** On an Azure SQL Database container +started without the `MSSQL_AAD_` variables it raises `Msg 40530`, above, and nothing about the +statement can be changed to make it run. + +## The name to put in the brackets + +The engine matches the token against this name, so a wrong one produces `Msg 33134` or `Msg 33131` +and reads as a permissions problem. + +| Principal | The name | +|---|---| +| A person | Their user principal name | +| A group | Its display name | +| An app registration or a user-assigned managed identity | Its display name | +| A system-assigned managed identity | The name of the Azure resource that owns it | +| A system-assigned identity on a deployment slot | `/slots/` | + +The slot form is the one nobody guesses, and an application that works in production and fails in a +staging slot is usually missing that user. + +## Which id goes where + +Three different identifiers, all GUIDs, all called something ending in ID. + +| Clause or property | Principal | The id it wants | +|---|---|---| +| `WITH OBJECT_ID` | Service principal | The **object id** from the enterprise application, not the app registration | +| `WITH SID`, `TYPE = E` | User | The user's **object id** | +| `WITH SID`, `TYPE = E` | Service principal or managed identity | The **application (client) id** | +| `WITH SID`, `TYPE = X` | Group | The group's **object id** | +| Connection string, user-assigned identity | Managed identity | The **client id**, except ODBC, which wants the object id outside a small set of hosts | + +The `sid` read back from `sys.database_principals` for an application converts to its +**application id**, so it can be compared with the identity the workload actually runs as. + +## What the server identity is for + +It is not the application's identity and it is not needed for a person to create users. + +Learn: for Azure SQL Database the server identity is optional, and required only when a Microsoft +Entra service principal creates or manages Entra users, groups or applications on the server. + +Either a system-assigned or a user-assigned identity can serve. A user-assigned one can hold the +Graph permissions once and be shared across servers, and it is not deleted with the server. + +Deleting whichever identity is serving this role leaves the server unable to reach Microsoft Graph, +and Entra authentication fails until a replacement is assigned and granted. + +## Verification queries + +The two catalogue queries live in the skill body, under "Check it worked", and run against the user +database, never `master`. What they will not tell you: + +- `sid_bytes` is `16` for a contained database user, whose SID is the object or client id itself, + and `18` for a user created from a Microsoft Entra server login, whose SID carries an `AADE` + suffix. The two do not match, so a login-based user cannot be correlated to its login by SID until + that suffix is removed. +- A row that exists with the wrong `entra_id` is indistinguishable from success until the + application fails, so compare the value rather than checking that the row is there. +- On a container with no `MSSQL_AAD_` variables both queries return nothing, because no `E` or `X` + principal can be created there. + +```sql +SELECT SERVERPROPERTY('IsExternalAuthenticationOnly'); -- 1 on, 0 off +``` + +## Sources + +Fetch these rather than trusting this summary when the details matter. Read on 2026-08-27, except +the last, read 2026-09-03. + +- `/azure/azure-sql/database/authentication-aad-service-principal`: the `33134` text and the impersonation mechanism. +- `/azure/azure-sql/database/authentication-aad-service-principal-tutorial`: the Graph permission list, and the rule that only Entra users can create Entra users. +- `/sql/relational-databases/security/authentication-access/authentication-microsoft-entra-create-users-with-nonunique-names`: `33131`, `37545` and the `WITH OBJECT_ID` rules. +- `/sql/t-sql/statements/create-user-transact-sql`: the syntax, `SID` and `TYPE`, and the permission requirement. +- `/azure/azure-sql/database/authentication-azure-ad-user-assigned-managed-identity`: the server identity rules and the no-lookup escape. +- `/azure/azure-sql/database/authentication-aad-overview`: the `sys.database_principals` property table and the 16 against 18 byte SID difference. +- `/azure/azure-sql/database/authentication-azure-ad-logins`: what a SQL admin or SQL user cannot execute. +- `/sql/relational-databases/errors-events/database-engine-events-and-errors-31000-to-41399`: the `40530` row, documented only as the batch rule. diff --git a/plugins/microsoft-sql-vscode/skills/entra-id-auth/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/entra-id-auth/skill.spec.jsonc new file mode 100644 index 0000000..d39f617 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/entra-id-auth/skill.spec.jsonc @@ -0,0 +1,148 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "entra-id-auth", + "domain": "identity-and-access", + "value": ["multi-step-with-silent-failures", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize"], + "correction": "Agents present Microsoft Entra ID setup for an application identity as a linear sequence of commands that either succeed or report an error, when the sequence carries steps that succeed while achieving nothing and one error whose text names the wrong component. Msg 33134 reads 'Principal could not be resolved', which sends the agent to check the principal name, but the principal is almost always correct and the actual fault is that the logical server has no identity holding Microsoft Graph read permission. That fault only appears when a service principal rather than a signed-in user runs CREATE USER, so the same statement works by hand and fails in a pipeline. Agents also assume any administrator can create the Entra database user, when a SQL authenticated connection cannot create one at all, and that CREATE USER FROM EXTERNAL PROVIDER run against master helps an application that connects to a user database, when the statement adds the user to whichever database the connection is on and reports success either way. Finally they invent per driver authentication keywords, most often by writing ActiveDirectoryDefault into an ODBC connection string, which has no such value. The whole class surfaces later as a login failure, 18456 naming a token-identified principal or 4060 refusing the database the login asked for, and an agent reading those as a network or connection-string fault edits the one place that cannot fix them.", + "value_probe": { + "task": "I run the local Azure SQL Database container for development. I started it with nothing but the licence acceptance and an administrator password, no other environment variables.\n\nOur deployment pipeline creates database users for application identities, and I want to test that step locally before it runs for real. So I connected as the administrator and ran exactly this, on its own, as the only thing in the batch:\n\n```sql\nCREATE USER [orders-api] WITH SID = 0x28C1D9A6E63D4B4CA0BF25F0C97F86F3, TYPE = E;\n```\n\nThe engine refuses it, and the message tells me that this statement has to be the only statement in the batch. It is the only statement in the batch. I have tried it in a fresh session with nothing before it, I have tried separating everything with GO, and I have tried running it through EXEC with the text in a variable. Same refusal every time. A plain CREATE USER with a password is accepted in the same session, so my connection and my permissions are fine.\n\nWhat is this actually telling me, and what do I need to do to test this step?", + "gate": "The answer attributes the refusal to the engine having no Microsoft Entra configuration rather than to the shape or contents of the batch, and directs the reader either to configure the container for Microsoft Entra or to test this step against a database in Azure, instead of rewriting the statement.", + "states": [ + "The message text names the wrong cause; the batch is genuinely correct and no rearrangement of it will help.", + "The engine is refusing because it has no Microsoft Entra configuration, so it cannot create an external principal of any kind.", + "The acceptance of a password based user in the same session proves the permissions and the connection are not involved.", + "Testing this step requires either configuring the container for Microsoft Entra or running it against an engine that already has that configuration." + ], + "refutes": [ + "The refusal is about batch structure, so separating the statements with a batch separator, moving the statement into its own batch, or wrapping it in dynamic execution will resolve it.", + "The statement is syntactically wrong, the security identifier is malformed, or the wrong user type letter was used, and correcting the syntax will make it succeed.", + "The administrator connection lacks a permission needed to create this user, and granting a server level permission or connecting as a different administrator will fix it.", + "The identity named does not exist in the directory, so the engine could not resolve it and refused." + ] + }, + "posture": ["provision", "admin", "execute"], + // Raised to ga on 2026-09-21, and this comment replaces one that named a + // defect rather than a result. Measured 2026-09-20 at ten runs per arm. The + // gate, which asks whether the answer blames the missing Microsoft Entra + // configuration rather than the shape of the batch, reads 9 of 10 with the + // skill against 3 of 10 without. A sixty point gap is the largest single + // result in this catalog. + // + // The verdict used to miss it, and that was a defect in the rule rather than + // in this skill. eval/run-value-eval.mjs built its verdict from the states and + // refutes checks only, so the gate's own effect was computed, written into + // every report, and never read: the gate could subtract, by returning NO GATE + // when the skill-loaded arm failed it, and could never add. A skill whose whole + // measured effect was getting the model to the right approach could not score + // better than NEITHER however large the gap. That was fixed on 2026-09-21 and + // the rule now lives in eval/lib/verdict.mjs, where its header says what a + // decisive gate contributes and what it still cannot do on its own. + // + // RE-DERIVED FROM THE SAME STORED COUNTS, not re-run. The verdict is LIFT, the + // one statement sitting between the thresholds at 10 of 10 against 8 of 10 is + // still reported as open, and the gate speaks over it because it falls short + // in the skill's favour rather than against it. All five probes pass, so + // scripts/check-maturity-drift.mjs computes a ga ceiling and this field is now + // at it. + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-to-azure-sql", + "connect-from-dotnet", + "connect-from-python", + "connect-from-typescript-and-node", + "diagnose-connection-errors", + "provision-azure-sql-db", + "build-app-on-azure-sql", + "deploy-app-to-azure", + "least-privilege-database-roles", + "entra-logins-and-server-roles", + "managed-identity-across-azure-services", + "entra-groups-and-access-reviews", + "secure-by-default-checklist" + ], + "validation": { + "target": "both", + "setup": [ + "a scratch database on the substrate under test, and an administrative connection to it", + "for the cloud-only assertions below, a logical server whose Microsoft Entra administrator can be set, and an Azure resource carrying a managed identity or an app registration to stand in for the application" + ], + "assert": [ + "the Microsoft Entra administrator is read back from the server before any CREATE USER statement is attempted", + "the connection running CREATE USER FROM EXTERNAL PROVIDER is authenticated with Microsoft Entra ID, not with a SQL login, and is on the user database rather than master", + "Msg 33134 is answered by checking the logical server identity and its Microsoft Graph permission, not by editing the principal name", + "a wrong or unknown principal name produces Msg 33130, not 33134 and not 33131. Measured on two logical servers, 2026-09-05 and 2026-09-08, with an unknown address, a bare guid and a non-existent application name", + "a display name that is duplicated in the tenant produces Msg 33131, measured 2026-09-08 with CREATE USER [Microsoft Graph] FROM EXTERNAL PROVIDER", + "WITH OBJECT_ID is a modifier on FROM EXTERNAL PROVIDER and not an alternative to it: written alone the statement is Msg 37546, and with a guid absent from the tenant it is Msg 37545. Both measured 2026-09-08", + "if Microsoft Entra-only authentication is enabled, the administrator was set first and the features it disables were stated", + "a login failure is answered at the layer that raised it: 18456 as a login, a secret or a missing database user, and 4060 as a missing user in the database that was asked for, never as a firewall rule or a connection-string edit", + "on the container, an 18456 naming an empty user is routed to connect-to-azure-sql as missing MSSQL_AAD_ configuration rather than answered with a database user", + "on the container, Msg 40530 raised by a lone CREATE USER ... TYPE = E or TYPE = X is read as missing Microsoft Entra configuration on the engine, never as a batch that has to be rewritten" + ], + "probes": [ + { + "sql": "CREATE USER [probe_no_login] WITHOUT LOGIN;", + "expect": { "succeeds": true }, + "why": "The control the two probes below are read against, and the reason Msg 40530 cannot be believed. One CREATE USER statement, alone in a batch, accepted. Measured 2026-09-03 on a container started with no MSSQL_AAD_ variables: it creates a SQL_USER. Whatever refuses the next two, it is not the shape of the batch." + }, + { + "sql": "CREATE USER [probe_entra_app] WITH SID = 0x11112222333344445555666677778888, TYPE = E;", + "expect": { "error": 40530 }, + "why": "The no-lookup form the skill hands every pipeline that cannot get Microsoft Graph permission, refused on an engine with no Microsoft Entra configuration, and refused by a message naming the wrong cause: Msg 40530 is 'The CREATE USER statement must be the only statement in the batch', and this is the only statement in the batch. Measured 2026-09-03 with zero MSSQL_AAD_ variables set; EXEC (@cmd) and GO separation return 40530 as well, so no rewriting of the batch escapes it. The id round trip this probe once asserted is a cloud claim and needs an Entra-configured engine. If a later build accepts this statement, or refuses it with a number that names the missing configuration, the probe goes red and the skill should then say so." + }, + { + "sql": "CREATE USER [probe_entra_group] WITH SID = 0x11112222333344445555666677778888, TYPE = X;", + "expect": { "error": 40530 }, + "why": "TYPE = X is the group half of the same form, refused identically on 2026-09-03, which puts the refusal on the external principal types rather than on E alone. Pinning both spellings means a build that starts accepting either one is caught." + }, + { + "cmd": "az sql server update --help 2>&1 | grep -q -- '--assign_identity -i'", + "where": "host", + "expect": { "succeeds": true }, + "why": "Guards the flag spelling in the Msg 33134 section. Measured 2026-09-03 on Azure CLI 2.90.0: the assign-identity flag is -i, long form --assign_identity with an underscore, and the hyphenated --assign-identity that every other Azure CLI flag would lead an agent to write exits with 'unrecognized arguments'. A script carrying the hyphenated form assigns no server identity and the next CREATE USER raises 33134 again. If a future CLI accepts the hyphen this probe goes red and the skill should then say both work." + }, + { + "cmd": "sqlcmd -S 127.0.0.1,$PROBE_PORT -d $PROBE_DB --authentication-method=NotARealMode -Q 'SELECT 1' 2>&1 | grep -q ActiveDirectoryManagedIdentity", + "where": "host", + "expect": { "succeeds": true }, + "why": "The skill tells a reader to ask the driver for its accepted set rather than recall it, and this settles that sqlcmd will. Measured 2026-09-03 on sqlcmd 1.10.0: an invalid value is rejected before any connection is opened, printing the full accepted list, so the check needs no reachable server and no Microsoft Entra tenant. Grepping for ActiveDirectoryManagedIdentity also pins the one spelling the skill's per-driver table gives for sqlcmd." + } + ] + }, + "triggering": { + "explicit": [ + "Set up Microsoft Entra authentication for my Azure SQL database", + "Give my app a managed identity and let it connect to Azure SQL Database without a password", + "How do I create a database user for a managed identity in Azure SQL Database", + "Turn on Microsoft Entra-only authentication and enforce it with Azure Policy" + ], + "implicit": [ + "I get Principal could not be resolved when my pipeline runs CREATE USER", + "I set the MSSQL_AAD variables on my local container, how do I let my app sign in with Entra ID now", + "My app connects fine from my laptop but gets login failed for user token-identified principal once it is deployed", + "CREATE USER FROM EXTERNAL PROVIDER says there is a duplicate display name", + "I am the server admin and I still cannot create the Entra user", + "How do I stop putting the SQL password in my configuration", + "Which Authentication value do I put in the connection string for a user-assigned managed identity", + "Cannot open database requested by the login. The login failed", + "Login failed for user, what do I do" + ], + "routing": [ + "the grant step of an azd deployment, where the database side user and roles have to be created", + "a login failure whose user name is an identity rather than a SQL login", + "a decision between a system-assigned and a user-assigned managed identity for database access", + "the managed identity exists in Azure but the database says login failed" + ], + "negative": [ + "how do I connect to Azure SQL Database from Node.js", + "add retry logic and size the connection pool for my Azure SQL data access layer", + "what does error 40613 mean, my first connection failed", + "the pre-login handshake failed and the connection timed out", + "deploy my app to Azure with azd and provision the infrastructure", + "create an Azure SQL database and open the firewall for my address", + "design the least-privilege database roles my reporting service needs", + "create a SQL login and password for the local Azure SQL Database container", + "how do I sign in to the Azure portal" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql deleted file mode 120000 index 1660d3c..0000000 --- a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/langchain-and-llamaindex-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/SKILL.md new file mode 100644 index 0000000..1d7162e --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/SKILL.md @@ -0,0 +1,356 @@ +--- +name: langchain-and-llamaindex-on-azure-sql +description: >- + Wires LangChain or LlamaIndex to Azure SQL Database from Python: the SQL toolkits, their text to + SQL prompts, the langchain-sqlserver vector store, and the guardrails neither framework + enforces. Use when someone asks to "use LangChain with Azure SQL", "build a SQL agent over the + database", "text to SQL", "SQLDatabaseToolkit", "NLSQLTableQueryEngine", "which LlamaIndex + vector store works with Azure SQL", or "make the SQL agent read only"; when a SQL agent keeps + generating LIMIT, its query checker approves a query the database refuses, or its schema tool + puts real rows into the prompt; or when a framework-created embedding table refuses CREATE + VECTOR INDEX or a metadata filter throws arithmetic overflow. Vector type and query shape are + vector-search-azure-sql, the cloud pipeline rag-on-azure-sql, the offline loop + rag-local-with-container, and drivers and token auth connect-from-python. +--- + +# LangChain and LlamaIndex on Azure SQL Database + +Both frameworks connect to this engine on the first try and then get three things wrong that the +connection test cannot show. This is not a text to SQL tutorial and not a vector search reference. + +**Measured 2026-08-28** by installing the packages, reading what ships, and running every statement +below against a local Azure SQL Database container (`SERVERPROPERTY('EngineEdition')` returns 5). +Every argument name was re-checked 2026-09-03 against the LangChain and LlamaIndex reference +documentation. The pinned versions move fast, so re-run the reference file before trusting one. + +## Reproduce it first, in two minutes + +```bash +python -m venv .venv && .venv/bin/pip install \ + "langchain==1.3.18" "langchain-community==0.4.2" "langchain-classic==1.0.8" \ + "langchain-sqlserver==1.0.1" "llama-index-core==0.14.24" "sqlalchemy==2.0.52" "pyodbc==5.3.0" +``` + +```python +import os, urllib.parse +from langchain_community.utilities import SQLDatabase +from langchain_community.agent_toolkits.sql.prompt import SQL_PREFIX + +odbc = urllib.parse.quote_plus( + "DRIVER={ODBC Driver 18 for SQL Server};SERVER=,;DATABASE=;" + "UID=;PWD=" + os.environ["SQL_PASSWORD"] + ";Encrypt=yes;TrustServerCertificate=yes") +db = SQLDatabase.from_uri(f"mssql+pyodbc:///?odbc_connect={odbc}") + +print(db.dialect) # mssql +print(SQL_PREFIX.format(dialect=db.dialect, top_k=10)) # "always limit your query to at most 10" +print(db.get_table_info()) # DDL plus three real rows per table +``` + +Three prints, three defects: the dialect is a string from the URL, the prompt tells the model to +limit, and the schema tool hands live rows to the model before anyone asked it a question. + +## The correction + +**1. The dialect is the string `mssql`, and the agent prompt still says "limit".** It comes from the +URL, not the server, so it is the same for the container and the cloud, and it is interpolated into +a prompt that never learned T-SQL. Section 1. + +**2. The framework-created vector table cannot take a vector index.** `langchain-sqlserver` 1.0.1 +creates it with `PRIMARY KEY NONCLUSTERED`, so the table is a heap and `CREATE VECTOR INDEX` is +refused with `Msg 42254`. The package holds no approximate search path at all, so retrieval is a +full scan for the life of the table, with correct results and latency that only grows. **LlamaIndex +has no Azure SQL Database vector store**, so an agent installs a neighbour and wires the wrong +database. Section 2. + +**3. Read-only, table scoping and no-DML are prompt text, not access control.** With +`include_tables` set to one table, the query tool still ran a `SELECT` against an excluded table and +returned a row; `db.run` executed an `INSERT` and a `CREATE`/`DROP` pair. Section 3. + +| Capability | LangChain | LlamaIndex | +|---|---|---| +| SQL toolkit over an existing schema | `SQLDatabaseToolkit`, four tools | `NLSQLTableQueryEngine` | +| Per-dialect text to SQL prompt | Only on the legacy chain path | **None.** One template per engine | +| Native `vector` column store | `langchain-sqlserver`, first party | **None.** Keep retrieval in SQL | +| Approximate vector search | Not implemented | Not applicable | +| Table info includes live rows | **Yes, three by default** | No, columns and types only | + +## 1. The dialect prompt, and the error that names the wrong token + +There are two prompt surfaces in LangChain and **the per-dialect fix reached only one of them.** + +| Surface | What it tells the model | Verdict | +|---|---|---| +| `SQL_PROMPTS['mssql']` in `langchain_classic.chains.sql_database.prompt`, used by `create_sql_query_chain` | "at most `{top_k}` results **using the TOP clause as per MS SQL**" | Correct | +| `SQL_PREFIX` in `langchain_community.agent_toolkits.sql.prompt`, the default `prefix` of `create_sql_agent` | "a syntactically correct `{dialect}` query ... always **limit** your query to at most `{top_k}` results" | Wrong, dialect-blind | +| The hub prompt `langchain-ai/sql-agent-system-prompt`, and the `system_prompt` written out in LangChain's own SQL agent guide | The same "always limit your query" wording | Wrong | +| `DEFAULT_TEXT_TO_SQL_TMPL` in `llama_index.core.prompts.default_prompts` | `{dialect}` only. No row cap, no pagination guidance, no per-dialect variant | Silent | + +The fix exists and sits on the chain path current documentation steers people away from. + +### The failure is loud in the wrong place + +Measured through `QuerySQLDatabaseTool`, which is what the model reads before it retries: + +| Query the model wrote | What comes back | +|---|---| +| `SELECT customer_id, full_name FROM customers LIMIT 10` | `Msg 102, Incorrect syntax near '10'` | +| `SELECT ... FROM customers ORDER BY full_name LIMIT 10` | `Msg 102, Incorrect syntax near 'LIMIT'` | +| `SELECT customer_id FROM customers OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY` | `Msg 102, Incorrect syntax near '0'` | + +**Without an `ORDER BY`, the error names the number and never mentions `LIMIT`.** The reason is +measured, not inferred: + +```sql +SELECT LIMIT.customer_id FROM dbo.customers LIMIT; -- succeeds, returns every row +``` + +T-SQL parses `LIMIT` as a table alias and then trips over the integer that follows it. A model +handed `Incorrect syntax near '10'` has no reason to remove a keyword the message does not mention, +so it edits the number, or the column list, and tries again. Two things make that worse: + +- The toolkit's `sql_db_query_checker` tool, which its own tool description says to call **before + every query**, uses a prompt listing `NOT IN` with nulls, `UNION` versus `UNION ALL`, `BETWEEN`, + type mismatches, quoting, argument counts, casts and join columns. **Dialect pagination is not on + the list.** The checker passes the query through. +- `QuerySQLDatabaseTool` calls `db.run_no_throw`, so the failure never raises in the application. It + becomes a string in the model's context and a longer trace. + +### Replace the prompt. Do not pass the default and hope + +`prefix` must still contain the `{dialect}` and `{top_k}` variables, so keep them and put the syntax +instruction in a sentence the dialect string is not carrying: + +```python +from langchain_community.agent_toolkits import create_sql_agent + +TSQL_PREFIX = """You are an agent designed to interact with an Azure SQL Database ({dialect}). +Write T-SQL. Return at most {top_k} rows with SELECT TOP ({top_k}), which is how this dialect caps +a result set. LIMIT does not exist in T-SQL and the parser reports it as a syntax error on the +number that follows it, not on the keyword. To page, write +ORDER BY OFFSET ROWS FETCH NEXT ROWS ONLY; the ORDER BY is mandatory. +Never issue INSERT, UPDATE, DELETE, DROP or any other DML statement.""" + +agent = create_sql_agent(llm, db=db, agent_type="tool-calling", prefix=TSQL_PREFIX, top_k=10) +``` + +LlamaIndex takes the same correction through `text_to_sql_prompt`, whose template needs `{dialect}`, +`{schema}` and `{query_str}`: + +```python +from llama_index.core import PromptTemplate +from llama_index.core.query_engine import NLSQLTableQueryEngine + +tsql_tmpl = PromptTemplate( + "Given an input question, write a syntactically correct T-SQL query for Azure SQL Database " + "({dialect}). Cap rows with SELECT TOP (n). LIMIT is not valid T-SQL. Page with " + "ORDER BY OFFSET ROWS FETCH NEXT ROWS ONLY.\n" + "Only use the tables below.\n{schema}\n\nQuestion: {query_str}\nSQLQuery: ") + +engine = NLSQLTableQueryEngine(sql_database=lidb, tables=["customers"], + text_to_sql_prompt=tsql_tmpl) +``` + +Keep the prompt short. The dialect rules themselves belong in `t-sql-correctness`, not in a prompt +string that then goes stale. + +## 2. Retrieval, and the index the vector store cannot have + +`langchain-sqlserver` is the supported LangChain vector store for this engine and it does use the +native type. `embedding_length` is required and is not derived from the embedding function; +`distance_strategy` defaults to cosine: + +```python +from langchain_sqlserver import SQLServer_VectorStore + +store = SQLServer_VectorStore(connection_string=f"mssql+pyodbc:///?odbc_connect={odbc}", + embedding_function=embeddings, embedding_length=1536, + table_name="lcv_docs") +``` + +The table it then creates, read back off the engine, with the dimension from `embedding_length`: + +```sql +CREATE TABLE lcv_docs ( + id UNIQUEIDENTIFIER NOT NULL, + custom_id VARCHAR(1000) NULL, + content_metadata json NULL, + content NVARCHAR(max) NOT NULL, + embeddings vector(1536) NOT NULL, + PRIMARY KEY NONCLUSTERED (id) -- this line is the problem +); +``` + +`PRIMARY KEY NONCLUSTERED` leaves the table a heap, and a heap cannot carry a vector index. Add the +clustered index immediately after the store first initialises, before it holds volume, because +adding it later rewrites the table's storage: + +```sql +CREATE VECTOR INDEX vi_lcv ON dbo.lcv_docs (embeddings) WITH (METRIC='cosine', TYPE='diskann'); +-- Msg 42254, Clustered index is required on table 'dbo.lcv_docs' to create a vector index. + +CREATE CLUSTERED INDEX ix_lcv_docs_cl ON dbo.lcv_docs (id); +SET QUOTED_IDENTIFIER ON; +CREATE VECTOR INDEX vi_lcv ON dbo.lcv_docs (embeddings) WITH (METRIC='cosine', TYPE='diskann'); +-- Msg 42266 once the clustered index clears the first gate: 100 non-null vectors are required. +``` + +`Msg 42266` is the expected second gate and proves the clustered index fixed the first. Without +`QUOTED_IDENTIFIER ON` the statement fails with `Msg 1934` before reaching either. + +### The emitted query is the exact-scan shape, and that is not configurable + +`similarity_search_with_score(k=2)` on a `vector(8)` store, captured off the connection with the +vector literal written out: + +```sql +SELECT TOP 2 lcv_docs.id, lcv_docs.custom_id, lcv_docs.content_metadata, lcv_docs.content, + VECTOR_DISTANCE('cosine', cast ('[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]' as vector(8)), + embeddings) AS distance +FROM lcv_docs ORDER BY distance ASC; +``` + +`TOP` and not `LIMIT`, because SQLAlchemy compiles the row cap for the dialect even when the prompt +does not. But the package contains **no `VECTOR_SEARCH`, no `WITH APPROXIMATE` and no +`CREATE VECTOR INDEX`** anywhere in its source, so there is no option to turn on. Below roughly tens +of thousands of rows, accept it. Above that, do the retrieval in SQL and keep the framework for +chunking, prompting and orchestration. Read `vector-search-azure-sql` before writing that query: the +shape that reaches the index is a different statement, not a tuning option. + +| Behaviour | Detail | +|---|---| +| A 3072 dimension model | `Msg 2717` at construction: the size exceeds the maximum allowed (1998). Decide the dimension budget first, which is `rag-on-azure-sql` | +| `embedding_length` mismatched to the function | `Msg 42204` at write time: the vector dimensions do not match | +| Metadata filters run inside the search query | `WHERE JSON_VALUE(content_metadata, ?) = ?`, parameterised, applied before the top-k. Good | +| **Numeric metadata filters cast to `NUMERIC(10, 2)`** | `{"ts": {"$gt": 1756500000}}` returns `Msg 8115`, arithmetic overflow converting nvarchar to numeric. The cast is on the stored column, so one oversized value poisons every numeric filter on that key, and it throws at read time, never at write. Keep epoch times, large ids and money out of metadata | +| Batch size is capped at 419 | Larger raises before any round trip | +| Auth falls back silently | No `Uid`/`Pwd` and no `Trusted_Connection=yes` switches to `DefaultAzureCredential`, so a typo in the credential keywords fails as a token acquisition, not as a bad password | + +### LlamaIndex has no Azure SQL Database vector store + +Checked on the package index and in the integrations tree on 2026-08-28. Do not let an agent install +a neighbouring package because the name looks close. The two honest options are to keep retrieval in +T-SQL and feed the rows in as nodes, or to use the SQL toolkit for structured questions and put the +semantic half where this catalog already covers it. Producing the embedding inside the engine is +`embeddings-and-external-models`. + +## 3. Read-only, no schema dumping, parameterised + +Treat all three as things to switch on. None of them is a default. + +### Read-only is a login, not a setting + +Neither framework has a read-only mode. `SQLDatabase.run` executed an `INSERT` that committed and a +`CREATE TABLE`/`DROP TABLE` pair in one call; LlamaIndex's `run_sql` executed an `UPDATE`. The +`DO NOT make any DML statements` line is a sentence in a prompt, so any instruction reaching the +model through retrieved content competes with it on equal terms. The control is a dedicated +least-privilege login. Measured through the query tool with that login: + +| Statement | Result | +|---|---| +| `SELECT TOP (2) full_name FROM customers` | Rows | +| `UPDATE customers SET city = N'Owned' WHERE customer_id = 1` | `Msg 229, The UPDATE permission was denied on the object 'customers'` | +| `DROP TABLE customers` | `Msg 3701, Cannot drop the table ..., because it does not exist or you do not have permission` | +| `EXEC sp_executesql N'SELECT 1'` | Runs. The tool does not restrict itself to `SELECT` text either | + +Open `azuresql-db-auth` before creating that login, because the grant is the whole guardrail. + +### `include_tables` scopes the prompt, not the connection + +Worth measuring yourself, because the parameter name suggests otherwise: + +```python +db = SQLDatabase.from_uri(uri, include_tables=["customers"], sample_rows_in_table_info=0) +print(db.get_usable_table_names()) # ['customers'] + +from langchain_community.tools.sql_database.tool import QuerySQLDatabaseTool +print(QuerySQLDatabaseTool(db=db).invoke("SELECT TOP (1) id FROM lcv_docs")) # returns the row +``` + +`include_tables` decides what the model is told about. It does not decide what the connection can +reach. Scope the login to the schema instead. + +### The schema tool returns live rows by default + +`sample_rows_in_table_info` defaults to `3`, so the schema tool returns the `CREATE TABLE` text +**plus three real rows**, every column, into the model's context and into whatever stores the trace. +On a customer table that is three real customers. + +```python +db = SQLDatabase.from_uri(uri, sample_rows_in_table_info=0) # schema only +db = SQLDatabase.from_uri(uri, custom_table_info={"customers": FAKE_DDL}) # or fake samples +``` + +Error text is a second channel: `run_no_throw` returns the driver message verbatim, database, schema +and object names included, as in `Msg 229` and `Msg 3701` above. + +### Parameterised means the parts you write + +The framework's own SQL binds its values. The query tool takes one argument, a SQL string the model +wrote, so a text to SQL agent is by construction a machine that executes generated SQL and there is +no parameterisation to add there; the containment is the login. In the code you write around the +framework, bind the values, and never build a query out of a chunk retrieved from the corpus. + +## Check it worked + +Four assertions against the wiring you are about to ship. The first two fail on a default install: + +```python +import os, re +from langchain_community.tools.sql_database.tool import QuerySQLDatabaseTool + +generated_sql = os.environ["AGENT_SQL"] # one query your agent actually produced +assert "SELECT TOP" in TSQL_PREFIX and "LIMIT does not exist" in TSQL_PREFIX +assert "3 rows from" not in db.get_table_info() +denied = QuerySQLDatabaseTool(db=db).invoke("UPDATE customers SET city = N'x' WHERE 1 = 0") +assert "permission was denied" in denied, denied +assert not re.search(r"\bLIMIT\b", generated_sql, re.I), generated_sql +print("prompt, schema exposure, write permission and generated SQL all check out") +``` + +Expected: the script prints that line and exits 0. A failure on the `get_table_info` assertion means +`sample_rows_in_table_info` is still `3`; a failure on the `denied` assertion means the agent's login +can write, which is the finding and not a test bug. Then confirm the vector table can carry an index +at all, with the password in `SQLCMDPASSWORD`: + +```bash +sqlcmd -S , -d -U -C -Q \ + "SELECT type_desc FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.lcv_docs') AND type_desc = 'CLUSTERED';" +``` + +Expected: one row reading `CLUSTERED`. An empty result set means the table is still the heap the +store created, and `CREATE VECTOR INDEX` on it will return `Msg 42254`. + +## Do not + +- Do not accept the default SQL agent prompt on this engine. It renders as `mssql`, tells the model + to limit, and the error it earns names the wrong token. +- Do not assume the per-dialect prompt applies to the agent. It exists only on the chain path. +- Do not conclude the query is fine because the checker tool approved it, and do not fix a `LIMIT` + failure by editing the number the error points at. +- Do not let the vector store create its table and then plan on a vector index later. The heap is + the blocker and the migration gets more expensive with every row. +- Do not expect approximate vector search from `langchain-sqlserver`. There is no code path. +- Do not install a LlamaIndex vector store package whose name merely resembles this engine's. +- Do not rely on `include_tables` or on a prompt sentence to stop a write. A login stops a write. +- Do not ship with `sample_rows_in_table_info` at its default and call the agent read only. +- Do not put an epoch timestamp or a large id in metadata that anything filters on numerically. +- Do not re-teach `VECTOR_DISTANCE`, `VECTOR_SEARCH` or the vector index here. Those are + `vector-search-azure-sql`. + +## References + +- Open [references/framework-prompts-and-vector-store.md](references/framework-prompts-and-vector-store.md) when a claim + above disagrees with what you are seeing, which after a version bump it will: it holds the install, + every probe script, the raw engine output and the reproduction steps. +- Read `vector-search-azure-sql` before writing the retrieval query yourself, for the `vector` type, + its restriction list and the query shape that reaches the index rather than scanning. +- Read `rag-on-azure-sql` before choosing `embedding_length`, because the dimension budget, the chunk + schema and the permission filter are decided there, and `rag-local-with-container` when the same + wiring must run offline with no cloud dependency and no keys. +- Read `t-sql-correctness` when the generated SQL is wrong in a way that is not pagination, + `connect-from-python` when the failure is the driver, the connection string or the token rather + than the framework, and `azuresql-db-auth` before creating the login section 3 depends on. +- Read [LangChain's SQL agent guide](https://docs.langchain.com/oss/python/langchain/sql-agent) to + check whether the shipped prompt has changed before trusting section 1, and + [LlamaIndex structured data](https://developers.llamaindex.ai/python/framework/understanding/putting_it_all_together/structured_data) + for where the prompt argument goes on the query engine. diff --git a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/references/framework-prompts-and-vector-store.md b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/references/framework-prompts-and-vector-store.md new file mode 100644 index 0000000..e573c17 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/references/framework-prompts-and-vector-store.md @@ -0,0 +1,297 @@ +# The shipped prompts, the vector store table and the guardrails, as measured + +Everything asserted in SKILL.md, with the command that produced it. Run on 2026-08-28 against a +local Azure SQL Database container, `SERVERPROPERTY('EngineEdition')` = 5, `@@VERSION` reporting +Microsoft SQL Azure. + +Two kinds of evidence, separated on purpose. **Read** is the shipped source of an installed package. +**Ran** is a statement that went to the engine and returned what is quoted. + +## Contents + +- [Setup](#setup) and [the four prompt surfaces](#read-the-four-prompt-surfaces) +- [The dialect string and the rendered prompt](#ran-the-dialect-string-and-the-rendered-prompt) +- [What LIMIT actually returns](#ran-what-limit-actually-returns) +- [The vector store table and its index](#ran-the-vector-store-table-and-its-index) +- [Dimensions, metadata filters and batching](#ran-dimensions-metadata-filters-and-batching) +- [No approximate search, no LlamaIndex integration](#read-no-approximate-search-no-llamaindex-integration) +- [The guardrails](#ran-the-guardrails) and [what was not verified](#what-was-not-verified) + +## Setup + +```bash +python -m venv .venv && .venv/bin/pip install \ + langchain langchain-community langchain-sqlserver llama-index-core sqlalchemy +.venv/bin/pip freeze | grep -Ei 'langchain|llama-index|sqlalchemy|pyodbc' +``` + +Resolved to `langchain==1.3.18`, `langchain-core==1.6.1`, `langchain-community==0.4.2`, +`langchain-classic==1.0.8`, `langchain-sqlserver==1.0.1`, `llama-index-core==0.14.24`, +`sqlalchemy==2.0.52`, `pyodbc==5.3.0`. + +The engine, and a table with fake rows: + +```sql +CREATE DATABASE appdb; +GO +CREATE TABLE dbo.customers ( + customer_id INT IDENTITY PRIMARY KEY, + full_name NVARCHAR(100) NOT NULL, + email NVARCHAR(200) NOT NULL, + ssn_last4 CHAR(4) NOT NULL, + city NVARCHAR(80) NOT NULL); +INSERT INTO dbo.customers (full_name, email, ssn_last4, city) VALUES + (N'Ada Example', N'ada@example.com', '0001', N'Springfield'), + (N'Bo Example', N'bo@example.com', '0002', N'Shelbyville'), + (N'Cy Example', N'cy@example.com', '0003', N'Ogdenville'), + (N'Di Example', N'di@example.com', '0004', N'North Haverbrook'); +``` + +Connection used by every Python probe, with the secret supplied from the environment: + +```python +odbc = urllib.parse.quote_plus( + "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,;DATABASE=appdb;" + "UID=example_username;PWD=" + os.environ["SQL_PASSWORD"] + ";TrustServerCertificate=yes") +uri = f"mssql+pyodbc:///?odbc_connect={odbc}" +``` + +## Read: the four prompt surfaces + +```bash +cat .venv/lib/python3.12/site-packages/langchain_community/agent_toolkits/sql/prompt.py +cat .venv/lib/python3.12/site-packages/langchain_classic/chains/sql_database/prompt.py +cat .venv/lib/python3.12/site-packages/langchain_community/tools/sql_database/prompt.py +sed -n '185,215p' .venv/lib/python3.12/site-packages/llama_index/core/prompts/default_prompts.py +``` + +- `SQL_PREFIX`, the agent prompt: `always limit your query to at most {top_k} results`. No T-SQL + branch, no `TOP`. +- `SQL_PROMPTS` in the classic chain module has eleven per-dialect entries. `SQL_PROMPTS['mssql']` + says `query for at most {top_k} results using the TOP clause as per MS SQL`, and + `create_sql_query_chain` selects it with `elif db.dialect in SQL_PROMPTS`. +- `QUERY_CHECKER`, behind `sql_db_query_checker`, lists eight mistakes: `NOT IN` with nulls, `UNION` + versus `UNION ALL`, `BETWEEN`, type mismatches, quoting, argument counts, casting, join columns. + Pagination syntax is not among them. +- `DEFAULT_TEXT_TO_SQL_TMPL` in LlamaIndex takes `{dialect}` and has no row cap sentence at all, and + there is no per-dialect table in that module. + +The hub prompt the toolkit docstring recommends was fetched and compared: + +```bash +curl -s https://api.smith.langchain.com/commits/langchain-ai/sql-agent-system-prompt/latest +``` + +Commit `31156d5f...`. Its template contains `always limit your query to at most {top_k} results`, +identical wording to `SQL_PREFIX`. + +## Ran: the dialect string and the rendered prompt + +```python +db = SQLDatabase.from_uri(uri) +db.dialect # -> 'mssql' +SQL_PREFIX.format(dialect=db.dialect, top_k=10) +``` + +Rendered output, second and third lines: + +```text +Given an input question, create a syntactically correct mssql query to run, then look at the +results of the query and return the answer. +Unless the user specifies a specific number of examples they wish to obtain, always limit your +query to at most 10 results. +``` + +`SQLDatabase.dialect` returns `self._engine.dialect.name`, fixed by the `mssql+pyodbc` URL and not +by the server, so it is the same string for the container and for the cloud. LlamaIndex on the same +database also returns `'mssql'`, and its default template renders `create a syntactically correct +mssql query` with no row-cap sentence following it. + +## Ran: what LIMIT actually returns + +Through `QuerySQLDatabaseTool.invoke`, which is what the model sees: + +| Statement | Returned | +|---|---| +| `SELECT customer_id, full_name FROM customers LIMIT 10` | `Incorrect syntax near '10'. (102)` | +| `SELECT customer_id, full_name FROM customers ORDER BY full_name LIMIT 10` | `Incorrect syntax near 'LIMIT'. (102)` | +| `SELECT customer_id FROM customers OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY` | `Incorrect syntax near '0'. (102)` | +| `SELECT TOP (5) customer_id FROM customers` | Rows | +| `SELECT customer_id FROM customers ORDER BY customer_id OFFSET 0 ROWS FETCH NEXT 5 ROWS ONLY` | Rows | + +Why the first message names the number: + +```sql +SELECT LIMIT.customer_id FROM customers LIMIT; -- succeeds, returns all four rows +``` + +`LIMIT` is a legal table alias in T-SQL, so with no `ORDER BY` the parser accepts it and fails on the +integer that follows. + +## Ran: the vector store table and its index + +Constructed `SQLServer_VectorStore` with a deterministic fake embedding function, +`embedding_length=8`, `table_name="lcv_docs"`, then read the DDL back off the connection: + +```sql +CREATE TABLE lcv_docs ( + id UNIQUEIDENTIFIER NOT NULL, + custom_id VARCHAR(1000) NULL, + content_metadata json NULL, + content NVARCHAR(max) NOT NULL, + embeddings vector(8) NOT NULL, + PRIMARY KEY NONCLUSTERED (id) +) +CREATE UNIQUE NONCLUSTERED INDEX idx_custom_id ON lcv_docs (custom_id) +``` + +`sys.indexes`: one `HEAP` row, one `NONCLUSTERED` primary key, one nonclustered unique index. Then: + +```sql +CREATE VECTOR INDEX vi_lcv ON dbo.lcv_docs(embeddings) WITH (METRIC='cosine', TYPE='diskann'); +-- Msg 42254, Level 16, State 1 +-- Clustered index is required on table 'dbo.lcv_docs' to create a vector index. + +CREATE CLUSTERED INDEX ix_lcv_docs_cl ON dbo.lcv_docs(id); +SET QUOTED_IDENTIFIER ON; +CREATE VECTOR INDEX vi_lcv ON dbo.lcv_docs(embeddings) WITH (METRIC='cosine', TYPE='diskann'); +-- Msg 42266, Level 16, State 1 +-- Cannot create a vector index. The table contains only 4 rows with non-null vectors, +-- but at least 100 are required for vector index creation. +``` + +The second message is the expected row-count gate, proving the clustered index cleared the first. +The session needs `QUOTED_IDENTIFIER ON`, or the statement fails with `Msg 1934` before either gate. + +The search SQL, captured with a `before_cursor_execute` listener on `similarity_search_with_score(k=2)`: + +```sql +SELECT TOP 2 lcv_docs.id, lcv_docs.custom_id, lcv_docs.content_metadata, lcv_docs.content, + lcv_docs.embeddings, + VECTOR_DISTANCE('cosine', cast ('[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]' as vector(8)), + embeddings) AS distance +FROM lcv_docs ORDER BY distance ASC; +``` + +`TOP`, not `LIMIT`, because SQLAlchemy compiles `.limit(k)` for the dialect. No `VECTOR_SEARCH`, +no `WITH APPROXIMATE`. The bound vector literal is written out above so the statement runs as +pasted; the capture printed the same query with the 8 floats of the query embedding. + +## Ran: dimensions, metadata filters and batching + +```python +from langchain_sqlserver import SQLServer_VectorStore + +SQLServer_VectorStore(connection_string=uri, embedding_function=fake8, + embedding_length=3072, table_name="lcv_big") +# The size (3072) given to the column 'embeddings' exceeds the maximum allowed (1998). (2717) + +vs = SQLServer_VectorStore(connection_string=uri, embedding_function=fake16, + embedding_length=8, table_name="lcv_mismatch") +vs.add_texts(["alpha"]) +# The vector dimensions 8 and 16 do not match. (42204) +``` + +`fake8` and `fake16` are deterministic embedding functions returning 8 and 16 floats respectively, +so the second block declares 8 dimensions and writes 16. + +Metadata filters, captured off the connection: + +```sql +-- filter={"src": "b"} +WHERE JSON_VALUE(lcv_docs.content_metadata, ?) = ? params ('$.src', 'b') + +-- filter={"n": {"$gt": 5}} +WHERE CAST(JSON_VALUE(lcv_num.content_metadata, ?) AS NUMERIC(10, 2)) > ? params ('$.n', 5) + +-- filter={"n": {"$in": [10]}} +WHERE JSON_VALUE(lcv_num.content_metadata, ?) IN (?) params ('$.n', '10') +``` + +Both the path and the value are bound. `$in` binds the value as a **string**, so it is an exact text +match on the JSON representation. + +The overflow, with metadata `{"ts": 1756000000}` and `{"ts": 1756900000}`: + +```python +vs.similarity_search("a", k=5, filter={"ts": {"$gt": 1756500000}}) +# Arithmetic overflow error converting nvarchar to data type numeric. (8115) +vs.similarity_search("a", k=5, filter={"ts": {"$gte": 1}}) +# Arithmetic overflow error converting nvarchar to data type numeric. (8115) +``` + +The second call proves the cast lands on the stored column, not on the bound value, so shrinking the +bound does not help. The rows insert without complaint; only the filter fails. + +Batch size is checked in `_validate_batch_size` against `MAX_BATCH_SIZE = 419`, raising before any +round trip. + +## Read: no approximate search, no LlamaIndex integration + +```bash +grep -n "VECTOR_SEARCH\|APPROXIMATE\|CREATE VECTOR INDEX\|diskann" \ + .venv/lib/python3.12/site-packages/langchain_sqlserver/vectorstores.py +# no matches in 1362 lines +``` + +The Entra fallback is in the same file: `_create_engine` registers `_provide_token` unless the +connection string carries a user and secret or `Trusted_Connection=yes`, and that helper calls +`DefaultAzureCredential`. + +For LlamaIndex, the package index returns 404 for every plausible name +(`llama-index-vector-stores-azuresql`, `-mssql`, `-azure-sql`, `-sqlserver`) while known-good +neighbours such as `llama-index-vector-stores-postgres` return 200, so the 404s are a real absence +and not a broken probe. The integrations directory confirms it: the Azure entries there are for +other services, none of them this engine. + +## Ran: the guardrails + +A dedicated login granted `db_datareader` and nothing else, through `QuerySQLDatabaseTool.invoke`: + +| Statement | Result | +|---|---| +| `SELECT TOP (2) full_name FROM customers` | `[('Ada Example',), ('Bo Example',)]` | +| `UPDATE customers SET city = N'Owned' WHERE customer_id = 1` | `The UPDATE permission was denied on the object 'customers', database 'appdb', schema 'dbo'. (229)` | +| `DROP TABLE customers` | `Cannot drop the table 'customers', because it does not exist or you do not have permission. (3701)` | +| `EXEC sp_executesql N'SELECT 1'` | `[(1,)]` | + +With the high-privilege login instead, `db.run` executed and committed an `INSERT` (row count 4 to +5) and a `CREATE TABLE dbo.scratch_probe (i INT); DROP TABLE dbo.scratch_probe;` pair in one call. +LlamaIndex's `run_sql` executed an `UPDATE` and the new value was visible on the next `SELECT`. + +Table scoping: + +```python +db = SQLDatabase.from_uri(uri, include_tables=["customers"], sample_rows_in_table_info=0) +db.get_usable_table_names() # ['customers'] +QuerySQLDatabaseTool(db=db).invoke("SELECT TOP (1) id FROM lcv_docs") +# [('CDD8DCEF-...',)] an excluded table, queried successfully +``` + +Schema exposure, `InfoSQLDatabaseTool.invoke("customers")`. By default the `CREATE TABLE` text then +a comment block headed `3 rows from customers table:` carrying all five columns of three real rows, +including the column standing in for sensitive data; with `sample_rows_in_table_info=0`, the +`CREATE TABLE` text only. LlamaIndex's `get_single_table_info("customers")` returns one line of +column names and types, no rows. + +Error handling differs between the frameworks and it matters for the retry loop: + +- LangChain: `QuerySQLDatabaseTool._run` calls `db.run_no_throw`, so nothing raises and the full + driver message, database, schema and object names included, reaches the model as a string. +- LlamaIndex: `run_sql` catches `ProgrammingError` and `OperationalError` and re-raises + `NotImplementedError("Statement ... is invalid SQL.\nError: ...")`. The driver message survives on + the second line, but the exception type does not, so an `except ProgrammingError` around it never + fires. + +## What was not verified + +- **Everything above ran against the container, not a cloud database.** The package behaviour is + substrate-independent because it is source. Engine messages carry their numbers so they can be + re-checked in the cloud in minutes, and the vector index numbers (`42254`, `42266`) match what + `vector-search-azure-sql` measured on both engines. +- **No model was called.** The prompts are quoted as rendered, not as followed. What is measured is + that the prompt says "limit", that the engine rejects `LIMIT`, and that the rejection names the + wrong token. +- **The checker tool was not exercised end to end**, only its prompt read. The claim is that its + list of mistakes omits pagination, which is a property of the text. +- **Re-ranking, streaming and the async paths** were not touched. diff --git a/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..25f9993 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/langchain-and-llamaindex-on-azure-sql/skill.spec.jsonc @@ -0,0 +1,260 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "langchain-and-llamaindex-on-azure-sql", + "domain": "ai-vector-and-rag", + "value": [ + "not-in-training-data", + "corrects-a-wrong-assumption", + "changes-too-fast-to-memorize" + ], + "correction": "Agents wire LangChain or LlamaIndex to Azure SQL Database by pointing a SQLAlchemy URL at it and taking the framework defaults, believing the framework knows the dialect and that its stated read-only, table-scoping and no-DML guardrails are enforced. Measured on 2026-08-28 by installing the packages, reading the shipped prompts and running every statement against an Azure SQL Database engine (langchain 1.3.18, langchain-community 0.4.2, langchain-classic 1.0.8, langchain-sqlserver 1.0.1, llama-index-core 0.14.24), four things are wrong at once. First, SQLDatabase.dialect is the literal string mssql and the agent prompt SQL_PREFIX, together with the hub prompt langchain-ai/sql-agent-system-prompt that the toolkit's own docstring recommends, still says to always limit the query to at most top_k results with no T-SQL branch; the per-dialect prompt that says TOP exists only on the legacy chain path, and LlamaIndex has no per-dialect prompt at all. The resulting failure misdirects: SELECT ... FROM t LIMIT 10 returns Msg 102 Incorrect syntax near '10' rather than near LIMIT, because T-SQL parses LIMIT as a table alias, and the sql_db_query_checker tool the agent is told to call before every query lists eight common mistakes of which dialect pagination is not one, so the repair loop edits the number and retries. Second, langchain-sqlserver creates its table with PRIMARY KEY NONCLUSTERED, leaving a heap that refuses CREATE VECTOR INDEX with Msg 42254, and the package contains no VECTOR_SEARCH and no WITH APPROXIMATE anywhere, so every retrieval is an exact full scan for the life of the table. Third, LlamaIndex ships no Azure SQL Database vector store at all, so an agent installs a neighbouring package for a different database. Fourth, include_tables scopes only what the model is told about and a query against an excluded table still returned a row, the no-DML instruction is a sentence in a prompt while db.run committed an INSERT and a CREATE and DROP pair, and sample_rows_in_table_info defaults to 3 so the schema tool puts three real rows of live data into the prompt. Being wrong costs retry loops the model cannot diagnose from the error it is given, a retrieval path that cannot be indexed without rebuilding the table, a numeric metadata filter that throws Msg 8115 arithmetic overflow at read time because the framework casts to NUMERIC(10, 2), and a tool surface that hands whatever the login can do to any text that reaches the model.", + "value_probe": { + "gate": "The answer puts every fix outside the model's behaviour: T-SQL aware prompting or dialect handling for the row cap, a change to the table's structure for the index refusal, and the login's own grants for the safety boundary, rather than more prompt wording and more retries.", + "task": "We pointed a LangChain SQL agent at our Azure SQL Database through a SQLAlchemy URL and gave it the standard toolkit. It answers simple questions fine, but as soon as a question needs a row cap it produces a query ending in LIMIT 10 and the database returns Incorrect syntax near '10'. The agent then changes the number and tries again, forever, and the checker tool it is told to call first never flags it.\n\nSeparately we used the vector store integration for that same database to hold about 900000 chunks. Retrieval is exact and slow, and when I tried to add the index that is supposed to speed that up the database refused it.\n\nWe also told our security team this setup is safe because we passed include_tables so it only sees three tables and the prompt says it is read only. Please explain the retry loop, the retrieval problem, and whether that safety claim holds.", + // 2026-09-19: the first statement used to read "The dialect the framework reports is correct + // while the shipped agent prompt still asks for LIMIT, so the error originates in the prompt + // rather than in a missing or wrong dialect." It measured where the LIMIT clause comes from, + // which the model already asserts 7 of 10 times with no skill loaded, so it had no room to show + // the skill adding anything and could only fall. That diagnosis is still enforced, by the gate + // and by the first refutes statement. The replacement measures the third question the task + // asks, whether the safety claim holds: the schema tool returns real rows by default, which no + // answer in the ten no-skill runs mentioned and all ten skill-loaded answers did. + "states": [ + "The tool that shows the agent the schema returns real rows out of those tables by default, so live data reaches the model's context and whatever records the trace before any question is asked.", + "The syntax error points at the number instead of the offending keyword, which is why the retry loop can never diagnose itself.", + "The table the vector store integration creates is a heap because its primary key is nonclustered, which is why the index is refused.", + "The integration's retrieval path is an exact full scan and offers no approximate form, so it stays a scan for the life of the table.", + "Scoping the table list limits only what the model is told about, not what a query the model writes is able to reach.", + "The read only or no DML instruction is prompt text rather than a permission, so the real boundary has to come from what the login is granted." + ], + "refutes": [ + "The framework detects the dialect and emits T-SQL row limiting, so the LIMIT clause is a model slip that better prompting or the query checker will resolve.", + "Passing include_tables prevents the agent from querying tables outside the list, so the omitted tables are genuinely inaccessible.", + "The read only instruction in the shipped prompt is enforced, so the agent cannot insert, update, create or drop anything.", + "LlamaIndex ships a vector store for Azure SQL Database that can be used directly.", + "The table the vector store integration creates supports the approximate index as it stands, with no change to the table." + ] + }, + "posture": [ + "read", + "write", + "execute" + ], + "maturity": "preview", + "applies_to": [ + "azure-sql-db", + "azure-sql-db-container" + ], + "related": [ + "vector-search-azure-sql", + "rag-on-azure-sql", + "rag-local-with-container", + "embeddings-and-external-models", + "t-sql-correctness", + "connect-from-python", + "prevent-sql-injection", + "sqlalchemy-azure-sql", + "chunking-strategies", + "azuresql-db-rag" + ], + "validation": { + "target": "both", + "setup": [ + "a Python environment with langchain, langchain-community, langchain-sqlserver and llama-index-core installed, and the pinned versions recorded", + "a database reachable with permission to create a table and an index, plus a second login granted read permission only", + "a table with a few fake rows, and an embedding function whose output dimension is known" + ], + "assert": [ + "the system prompt handed to the agent or the query engine names T-SQL and SELECT TOP (n) explicitly, and does not interpolate the dialect string into the sentence describing syntax", + "no generated query in the traces contains a LIMIT clause, and a test fails if one does", + "an UPDATE issued through the agent's own connection is denied by the engine rather than by the prompt", + "a query against a table outside include_tables is refused by permissions, not merely absent from the schema tool output", + "sample_rows_in_table_info is 0, or the sample rows come from fake data supplied through custom_table_info", + "the vector store's table carries a clustered index if a vector index is ever intended", + "the declared embedding_length equals the dimension requested from the embedding model and is 1998 or fewer", + "no metadata key that anything filters on numerically holds a value larger than NUMERIC(10, 2) can carry", + "the framework versions are pinned and recorded, and the prompt and vector store behaviour were re-checked at the last upgrade" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_ll_alias (customer_id int NOT NULL PRIMARY KEY, full_name nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_ll_alias (customer_id, full_name) VALUES (1, N'Ada Example'), (2, N'Bo Example');\nGO\nSELECT COUNT(LIMIT.customer_id) AS rows_seen FROM dbo.probe_ll_alias LIMIT;", + "expect": { + "rows": [ + [ + 2 + ] + ] + }, + "why": "The reason the repair loop never removes the keyword. LIMIT is not reserved in T-SQL, so the parser accepts it as a table alias and this statement succeeds over both rows. That is why the failure in the next probe names the integer that follows LIMIT and never names LIMIT itself, and why a model handed 'Incorrect syntax near 10' edits the number instead. Measured 2026-08-28." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_limit (customer_id int NOT NULL PRIMARY KEY, full_name nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_ll_limit (customer_id, full_name) VALUES (1, N'Ada Example'), (2, N'Bo Example');\nGO\nSELECT customer_id, full_name FROM dbo.probe_ll_limit LIMIT 10;", + "expect": { + "error": 102 + }, + "why": "The query the default agent prompt asks for, refused by the engine. SQL_PREFIX in langchain_community, the hub prompt langchain-ai/sql-agent-system-prompt and the system prompt written out in LangChain's own SQL agent guide all say to always limit the query to at most top_k results, with no T-SQL branch. Msg 102 is the engine's incorrect-syntax error, measured 2026-08-28 and reported by the tool as Incorrect syntax near '10'." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_offset (customer_id int NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_ll_offset (customer_id) VALUES (1), (2);\nGO\nSELECT customer_id FROM dbo.probe_ll_offset OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY;", + "expect": { + "error": 102 + }, + "why": "The second thing a model tries after LIMIT fails, and it fails too, which is why the replacement prompt in this skill states that the ORDER BY is mandatory rather than just naming OFFSET and FETCH. Msg 102 measured 2026-08-28; the same statement with ORDER BY customer_id in front of OFFSET returns rows." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_lcv (\n id uniqueidentifier NOT NULL,\n custom_id varchar(1000) NULL,\n content_metadata json NULL,\n content nvarchar(max) NOT NULL,\n embeddings vector(8) NOT NULL,\n PRIMARY KEY NONCLUSTERED (id));\nINSERT INTO dbo.probe_ll_lcv (id, content, embeddings) VALUES\n (NEWID(), N'alpha', '[1,2,3,4,5,6,7,8]'), (NEWID(), N'beta', '[2,3,4,5,6,7,8,9]'),\n (NEWID(), N'gamma', '[3,4,5,6,7,8,9,1]'), (NEWID(), N'delta', '[4,5,6,7,8,9,1,2]');\nGO\nSET QUOTED_IDENTIFIER ON;\nCREATE VECTOR INDEX vi_probe_ll_lcv ON dbo.probe_ll_lcv (embeddings) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42254 + }, + "why": "The table langchain-sqlserver 1.0.1 creates, reproduced as DDL exactly as it was read back off the engine on 2026-08-28, including the PRIMARY KEY NONCLUSTERED that leaves it a heap. Msg 42254, clustered index is required to create a vector index, is what the package's own table earns, and it is the whole reason this skill says to add the clustered index before the table holds volume. If this ever succeeds, the package changed its DDL and section 2 needs re-reading." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_lcv_cl (\n id uniqueidentifier NOT NULL,\n custom_id varchar(1000) NULL,\n content nvarchar(max) NOT NULL,\n embeddings vector(8) NOT NULL,\n PRIMARY KEY NONCLUSTERED (id));\nINSERT INTO dbo.probe_ll_lcv_cl (id, content, embeddings) VALUES\n (NEWID(), N'alpha', '[1,2,3,4,5,6,7,8]'), (NEWID(), N'beta', '[2,3,4,5,6,7,8,9]'),\n (NEWID(), N'gamma', '[3,4,5,6,7,8,9,1]'), (NEWID(), N'delta', '[4,5,6,7,8,9,1,2]');\nGO\nCREATE CLUSTERED INDEX ix_probe_ll_lcv_cl ON dbo.probe_ll_lcv_cl (id);\nGO\nSET QUOTED_IDENTIFIER ON;\nCREATE VECTOR INDEX vi_probe_ll_lcv_cl ON dbo.probe_ll_lcv_cl (embeddings) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42266 + }, + "why": "The same table with one CREATE CLUSTERED INDEX added, and the refusal moves on to the ordinary row-count gate. That change of error number is the evidence that the clustered index is the fix for Msg 42254 and not a coincidence: 42266 is the 100 non-null vectors minimum that vector-search-azure-sql owns, reached here with four rows. Measured 2026-08-28." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_meta (id int NOT NULL PRIMARY KEY, content_metadata nvarchar(max) NOT NULL);\nINSERT INTO dbo.probe_ll_meta (id, content_metadata) VALUES (1, N'{\"ts\": 1756000000}'), (2, N'{\"ts\": 1756900000}');\nGO\nSELECT id FROM dbo.probe_ll_meta\nWHERE CAST(JSON_VALUE(content_metadata, '$.ts') AS NUMERIC(10, 2)) > 1756500000;", + "expect": { + "error": 8115 + }, + "why": "The WHERE clause langchain-sqlserver 1.0.1 emits for a numeric metadata filter, with the NUMERIC(10, 2) cast it hardcodes, reproduced against the same values. NUMERIC(10, 2) tops out below 100 million, so any epoch timestamp overflows. Microsoft Learn lists Msg 8115 as arithmetic overflow error converting to a data type; the package's own filter earns it, measured 2026-08-28. The cast lands on the stored column, so a single oversized value poisons every numeric filter on that key, and it throws at read time and never at write time." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_readonly (customer_id int NOT NULL PRIMARY KEY, city nvarchar(40) NOT NULL);\nINSERT INTO dbo.probe_ll_readonly (customer_id, city) VALUES (1, N'Springfield');\nCREATE USER probe_ll_reader WITHOUT LOGIN;\nGRANT SELECT ON dbo.probe_ll_readonly TO probe_ll_reader;\nGO\nEXECUTE AS USER = 'probe_ll_reader';\nUPDATE dbo.probe_ll_readonly SET city = N'Owned' WHERE customer_id = 1;", + "expect": { + "error": 229 + }, + "why": "The only guardrail in section 3 that is enforced by anything. The DO NOT make any DML statements line in SQL_PREFIX is prompt text and db.run committed an INSERT and a CREATE and DROP pair straight through it, so the write has to be stopped by the grant instead. Msg 229, the UPDATE permission was denied on the object, is what the agent's login must earn, measured 2026-08-28 through the query tool. Microsoft Learn documents CREATE USER WITHOUT LOGIN and EXECUTE AS USER on Azure SQL Database, which is how the probe takes the agent's identity without needing a second connection." + }, + { + "sql": "CREATE TABLE dbo.probe_ll_scan (id int NOT NULL PRIMARY KEY CLUSTERED, content nvarchar(50) NOT NULL, embeddings vector(3) NOT NULL);\nINSERT INTO dbo.probe_ll_scan (id, content, embeddings) VALUES (1, N'a', '[1,2,3]'), (2, N'b', '[1,2,4]'), (3, N'c', '[9,9,9]');\nGO\nSELECT id FROM (\n SELECT TOP 2 lcv.id, VECTOR_DISTANCE('cosine', cast ('[1,2,3]' as vector(3)), embeddings) AS distance\n FROM dbo.probe_ll_scan AS lcv ORDER BY distance ASC) AS nearest\nORDER BY id;", + "expect": { + "rows": [ + [ + 1 + ], + [ + 2 + ] + ] + }, + "why": "The query langchain-sqlserver 1.0.1 actually emits for similarity_search_with_score, captured off the connection on 2026-08-28 and reduced to three dimensions. It confirms two things at once: SQLAlchemy compiles the row cap as TOP and not LIMIT even though the agent prompt does not, and the shape is ORDER BY VECTOR_DISTANCE, which vector-search-azure-sql measures as a full scan that never reaches a vector index. Correct rows are exactly why nobody notices." + } + , + { + "sql": "CREATE TABLE dbo.probe_lc_cust (customer_id int, full_name nvarchar(50));\nGO\nSELECT customer_id, full_name FROM dbo.probe_lc_cust LIMIT 10;", + "expect": { + "error": 102 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. LIMIT is the first thing a text to SQL chain emits, and it does not parse here. This is the number an agent has to be able to read back to a dialect mistake rather than to a schema one. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_off (customer_id int);\nGO\nSELECT customer_id FROM dbo.probe_lc_off OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY;", + "expect": { + "error": 102 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The correct looking repair is also refused, because OFFSET needs an ORDER BY. Both attempts produce Msg 102 and only the second is nearly right. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT CAST(N'1756500000' AS numeric(10,2)) AS x;", + "expect": { + "error": 8115 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A numeric metadata filter cast to NUMERIC(10, 2) overflows on an epoch timestamp, which is what turns a working filter into an error the day someone filters on a date. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_perm (customer_id int, city nvarchar(50));\nGO\nCREATE USER probe_lc_low WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_lc_low';\nUPDATE dbo.probe_lc_perm SET city = N'Owned' WHERE customer_id = 1;\nREVERT;", + "expect": { + "error": 229 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The permission wall a read only chain has to be behind, asserted rather than assumed. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_drop (id int);\nGO\nCREATE USER probe_lc_low2 WITHOUT LOGIN;\nGO\nEXECUTE AS USER = 'probe_lc_low2';\nDROP TABLE dbo.probe_lc_drop;\nREVERT;", + "expect": { + "error": 3701 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And the DDL wall. Note the wording: the engine says the table does not exist or you do not have permission, so a chain that reports the message verbatim leaks the object name and tells the caller nothing true. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_heap (id int NOT NULL, v vector(3) NOT NULL);\nGO\nCREATE VECTOR INDEX probe_lc_heap_ix ON dbo.probe_lc_heap (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42254 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The vector store the package creates has no clustered index, so the index a reader then tries to add is refused. This is the first of the two gates the skill walks through. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_99 (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_lc_99 (v) SELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 99);\nGO\nCREATE VECTOR INDEX probe_lc_99_ix ON dbo.probe_lc_99 (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42266 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The second gate, and the skill's own words are that reaching it proves the clustered index fixed the first. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_3072 (v vector(3072));", + "expect": { + "error": 2717 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A 3072 dimension model does not fit, which decides the dimension budget before any chain is written. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_lc_len (id int, v vector(3) NOT NULL);\nGO\nINSERT INTO dbo.probe_lc_len VALUES (1, N'[1,2,3,4]');", + "expect": { + "error": 42204 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. embedding_length mismatched to the function fails at write time, not at configuration time, so the mistake survives setup and appears on the first document. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SET QUOTED_IDENTIFIER OFF;\nCREATE TABLE dbo.probe_lc_qi (id int NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_lc_qi (id, v) SELECT value, CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nSET QUOTED_IDENTIFIER OFF;\nCREATE VECTOR INDEX probe_lc_qi_ix ON dbo.probe_lc_qi (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 1934 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Without QUOTED_IDENTIFIER ON the index build fails before reaching either gate, and the message names indexed views and computed columns rather than the setting, which is why the skill says check it first. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Use the langchain-and-llamaindex-on-azure-sql skill", + "Wire LangChain up to my Azure SQL Database", + "Build a SQL agent over Azure SQL Database with the SQLDatabaseToolkit", + "Use LlamaIndex to answer questions over tables in Azure SQL Database", + "Which LangChain vector store should I use with Azure SQL Database", + "Set up text to SQL against Azure SQL Database in Python" + ], + "implicit": [ + "the agent keeps generating LIMIT and the database rejects it", + "my SQL agent gets incorrect syntax near a number and retries forever without fixing it", + "why does the query checker approve a query the database then refuses", + "I cannot create a vector index on the table the vector store made for me", + "is there a LlamaIndex integration for Azure SQL Database", + "how do I stop the SQL agent from being able to write to the database", + "the schema tool is putting real customer rows into the prompt", + "my metadata filter started failing with arithmetic overflow", + "does include_tables actually stop the agent reading other tables" + ], + "routing": [ + "add a natural language question answering feature over an existing Azure SQL Database schema", + "port a LangChain application from another database to Azure SQL Database", + "put a chat interface in front of Azure SQL Database using a Python agent framework" + ], + "negative": [ + "What are the restrictions on the vector data type in Azure SQL Database", + "Why does my similarity query scan the whole table instead of using the vector index", + "Show me the syntax for VECTOR_DISTANCE and the distance metrics", + "Build a retrieval augmented generation pipeline over my document library on Azure SQL Database", + "How do I keep embeddings in sync when the source rows change", + "Run a local RAG prototype offline against the Azure SQL Database container", + "Build a RAG scenario on the container with a local embedding model", + "Create an external model and call AI_GENERATE_EMBEDDINGS from T-SQL", + "Which permissions does an external model endpoint need and how does the allowlist work", + "Install the ODBC driver and write a Python connection string for Azure SQL Database", + "Get a Microsoft Entra token in Python and hand it to the driver", + "Rewrite this PostgreSQL query as T-SQL", + "Design the schema for my orders and customers tables" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection deleted file mode 120000 index 283fe79..0000000 --- a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection +++ /dev/null @@ -1 +0,0 @@ -../../../skills/prevent-sql-injection \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/SKILL.md b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/SKILL.md new file mode 100644 index 0000000..4d96a9d --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/SKILL.md @@ -0,0 +1,211 @@ +--- +name: prevent-sql-injection +description: >- + Handles SQL injection on Azure SQL Database beyond parameterisation: a typed sp_executesql + parameter matches nothing where the same input concatenated into EXEC() returns every row; + QUOTENAME returns NULL above 128 characters, so the batch built from it becomes NULL and does + nothing; a dynamic ORDER BY built from one CASE over mixed types fails only for the sort key on + the lower-precedence branch; dynamic SQL breaks the ownership chain, so EXECUTE AS decides what + it may touch; and Always Encrypted refuses a literal (Msg 206). Use for a general injection + question or a pre-production review, when a QUOTENAME-built statement returns and raises + nothing, when a sort-by-column feature throws an operand type clash for one column only, when a + procedure works until its query becomes dynamic, or when an encrypted column will not take a + literal. Row level tenant isolation is rls-multi-tenant. +--- + +# Prevent SQL injection: the value, the identifier, and the context it runs under + +Parameterising a value is the part an agent already gets right. Confirm it in a sentence and spend +the answer on what a parameter cannot reach: an identifier, the type of a runtime-chosen sort +expression, the permissions a dynamic batch runs under, and an encrypted column. + +Measured 2026-08-29 against an engine reporting `EngineEdition` 5 and Edition `SQL Azure`. +`QUOTENAME`, `PARSENAME`, `sp_executesql`, ownership chaining and Always Encrypted were checked +against Microsoft Learn on 2026-09-03; flags from `sqlcmd` 1.10.0 help. + +## 1. A parameter is a value, a concatenation is code, and the difference is measurable + +Do not assert this. Run it. Both statements get the same string. + +```sql +CREATE TABLE dbo.Users (id int NOT NULL PRIMARY KEY, name nvarchar(50) NOT NULL); +INSERT INTO dbo.Users (id, name) VALUES (1, N'ann'), (2, N'bob'); + +DECLARE @input nvarchar(100) = N'ann'' OR 1=1--'; + +-- Typed parameter: the input is one value, and no row holds it. Returns 0. +EXEC sp_executesql N'SELECT COUNT(*) AS matched FROM dbo.Users WHERE name = @n', + N'@n nvarchar(100)', @n = @input; + +-- Same input concatenated: the input is now part of the statement. Returns 2. +DECLARE @sql nvarchar(max) = + N'SELECT COUNT(*) AS matched FROM dbo.Users WHERE name = N''' + @input + N''''; +EXEC (@sql); +``` + +Zero against two, from one string. Learn's rule for `@params` is the discipline in one line: values +can only be constants or variables, never expressions built with operators. + +## 2. No parameter binds an identifier, and QUOTENAME's ceiling is a NULL + +`EXEC sp_executesql N'SELECT * FROM @t', N'@t sysname', @t = @name` cannot work: `@t` is a value, +not a table. A dynamic identifier has to enter the SQL text, so `QUOTENAME`'s documented edges are +load bearing. Learn: `character_string` is **sysname**, and an input over 128 characters returns +`NULL`, as does an unacceptable quote character. Escaping is not the defect: an embedded `]` is +doubled correctly. Crossing 128 is, because a `NULL` in a batch makes the whole batch `NULL`: + +```sql +DECLARE @n129 nvarchar(200) = REPLICATE(N'a', 129); +SELECT LEN(QUOTENAME(REPLICATE(N'a', 128))) AS len_at_128, -- 130 + IIF(QUOTENAME(@n129) IS NULL, 1, 0) AS null_at_129, -- 1 + IIF(QUOTENAME(N'tbl', N'#') IS NULL, 1, 0) AS null_bad_quote; -- 1 + +DECLARE @sql nvarchar(max) = N'SELECT * FROM ' + QUOTENAME(@n129); +EXEC sp_executesql @sql; +SELECT IIF(@sql IS NULL, 1, 0) AS batch_was_null, @@ERROR AS err; -- 1, 0 +``` + +`EXEC (@sql)` on the same `NULL` behaves identically: this is the `NULL` batch, not one calling +convention. Nothing reports it at any severity: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 \ + -Q "DECLARE @s nvarchar(max)=N'SELECT 1 FROM '+QUOTENAME(REPLICATE(N'x',129)); EXEC sp_executesql @s; SELECT @@ERROR AS err;" +``` + +`-b` exits non-zero at severity 11 and above and `-m-1` lowers the print level, but there is no +message at any level: `err` is 0 and so is the exit code. The `NULL` has to be the check. + +**Neither helper checks that the object exists.** `QUOTENAME` brackets a string and `PARSENAME` +splits a qualified name; Learn says each part is **sysname**, a part over 256 bytes comes back +`NULL`, and it does not indicate whether an object of that name exists. Only a catalog lookup +decides existence, so resolve the caller's string against the catalog and build from what it +returned: + +```sql +DECLARE @requested nvarchar(400) = N'dbo.Users', @safe nvarchar(300); +SELECT @safe = QUOTENAME(s.name) + N'.' + QUOTENAME(o.name) +FROM sys.objects AS o JOIN sys.schemas AS s ON s.schema_id = o.schema_id +WHERE s.name = PARSENAME(@requested, 2) AND o.name = PARSENAME(@requested, 1) AND o.type = 'U'; +IF @safe IS NULL THROW 50001, 'No row in the catalog matched that name.', 1; + +DECLARE @sql nvarchar(max) = N'SELECT COUNT(*) AS n FROM ' + @safe; +EXEC sp_executesql @sql; +``` + +The rejection there is the lookup's, not `QUOTENAME`'s: an over-long name matches no row because no +table can carry one. Where the legal identifiers are known in advance, an allowlist is simpler. + +## 3. One CASE over mixed types in a dynamic ORDER BY fails for some sort keys and not others + +A single `CASE` forces every branch to one common type chosen by type precedence, not by the branch +the sort key selects. Measured on `dbo.T (i int, d date, s nvarchar(50))`: + +| Branches | sort key selects | Result | +|---|---|---| +| `THEN i` / `THEN d` | anything, including neither | `Msg 206`, operand type clash, int against date | +| `THEN s` / `THEN d` | `d`, the higher-precedence side | runs clean | +| `THEN s` / `THEN d` | `s`, the lower-precedence side | `Msg 241`, conversion failed from character string | +| `THEN i` / `THEN s` | `s`, the lower-precedence side | `Msg 245`, conversion failed converting nvarchar to int | + +`int` and `date` have no implicit conversion, so that pairing is refused at compile time and a +surrounding `BEGIN TRY` does not catch it. The string pairings are the sharper trap: the identical +statement passes for one sort key and fails for another, so it survives a test against whichever +column the author tried. One `CASE` per candidate column, each carrying a single type, has no common +type to coerce toward; non-matching columns evaluate to `NULL` and tie, so this composes with +`OFFSET`/`FETCH` and a direction toggle: + +```sql +DECLARE @col sysname = N'created', @dir char(1) = N'A'; +SELECT id, name, created FROM dbo.Grid +ORDER BY CASE WHEN @col = N'id' AND @dir = N'A' THEN id END ASC, + CASE WHEN @col = N'id' AND @dir = N'D' THEN id END DESC, + CASE WHEN @col = N'created' AND @dir = N'A' THEN created END ASC +OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY; +``` + +## 4. Dynamic SQL breaks the ownership chain, so EXECUTE AS is part of the answer + +Learn: dynamic SQL in procedural code breaks the ownership chain, so the engine checks the caller's +permissions against every object the statement touches. A procedure a caller could run yesterday +stops working the day its query becomes dynamic, and parameterising does not bring it back. + +```sql +CREATE TABLE dbo.Ledger (id int NOT NULL PRIMARY KEY); +INSERT INTO dbo.Ledger (id) VALUES (1); +CREATE USER app_reader WITHOUT LOGIN; +GO +CREATE PROCEDURE dbo.CountLedgerDynamic AS + EXEC sp_executesql N'SELECT COUNT(*) AS n FROM dbo.Ledger'; +GO +CREATE PROCEDURE dbo.CountLedgerAsOwner WITH EXECUTE AS OWNER AS + EXEC sp_executesql N'SELECT COUNT(*) AS n, CURRENT_USER AS running_as FROM dbo.Ledger'; +GO +GRANT EXECUTE ON dbo.CountLedgerDynamic TO app_reader; +GRANT EXECUTE ON dbo.CountLedgerAsOwner TO app_reader; +GO +EXECUTE AS USER = 'app_reader'; +EXEC dbo.CountLedgerDynamic; -- Msg 229, SELECT permission denied on 'Ledger' +EXEC dbo.CountLedgerAsOwner; -- 1, dbo +REVERT; +``` + +Learn names two remedies. `EXECUTE AS` replaces the caller's permissions for the whole module, +nested modules inherit the proxy context, and the identity functions report the proxy, which is what +`running_as` returning `dbo` shows, so anything keyed to the caller must arrive as a parameter. +Certificate signing merges the certificate user's permissions with the caller's and leaves the +execution context alone, at the cost of re-signing on every change. Open `rls-multi-tenant` before +putting `EXECUTE AS` on a module touching a table under a security policy: the proxy context is what +a tenant predicate reads. Neither remedy substitutes for parameterising. + +## 5. On an Always Encrypted column a literal does not run at all + +Learn lists it under Always Encrypted's limitations: comparing an encrypted column to a literal, or +inserting one, fails with `Msg 206, Level 16, State 2, Operand type clash`. Only a value bound as a +parameter over a connection with `Column Encryption Setting=Enabled` is encrypted before it leaves +the driver, so parameterising is the only thing that returns a row. Documented, not reproduced here: +it needs a column master key provisioned outside the database. + +## Check it worked + +```sql +-- Every module that builds SQL, and the context it runs under. A NULL principal id means the +-- caller's permissions apply to whatever the dynamic statement touches. +SELECT o.name, m.execute_as_principal_id +FROM sys.sql_modules AS m JOIN sys.objects AS o ON o.object_id = m.object_id +WHERE m.definition LIKE N'%sp_executesql%' OR m.definition LIKE N'%EXECUTE (%'; +``` + +```sql +-- No dynamic batch executes while NULL. Expect the rejection, not zero rows. +DECLARE @batch nvarchar(max) = N'SELECT * FROM ' + QUOTENAME(REPLICATE(N'x', 129)); +IF @batch IS NULL SELECT 'rejected before execution' AS result ELSE EXEC sp_executesql @batch; +``` + +Then re-run section 1's pair against the real table: parameterised matches zero rows, concatenated +every row. Run the generated `ORDER BY` once per sort key a caller may send, not only the one +tested: any raising `Msg 206`, `241` or `245` means a mixed-type `CASE` is still there. + +## Do not + +- Do not present `QUOTENAME` as sufficient for a caller-supplied identifier without checking its + result for `NULL`, and do not say it truncates a long name. It returns `NULL`. +- Do not say either helper checks that an object exists. Neither reads the catalog. +- Do not build a caller-chosen `ORDER BY` from one `CASE` spanning columns of different types, even + when the columns tried in testing share a type. +- Do not add `WITH EXECUTE AS OWNER` to clear a permission error without saying what the module can + now reach and that the caller's identity is no longer visible inside it. +- Do not tell a developer an agent-composed statement is made safe by parameterising the one value + inside it. The model authored the whole statement, so the control is the permission boundary its + connection runs under. +- Do not warn about `xp_cmdshell`, linked servers or `OPENROWSET` against arbitrary providers. None + exist on Azure SQL Database to escalate to. + +## References + +- Open [the measured runs](references/quotename-null-and-order-by-clashes.md) when a claim above + disagrees with what you see, or to look up the message a column-type pairing produces. +- Microsoft Learn, if a number above looks stale: + [QUOTENAME](https://learn.microsoft.com/sql/t-sql/functions/quotename-transact-sql), + [PARSENAME](https://learn.microsoft.com/sql/t-sql/functions/parsename-transact-sql), + [secure dynamic SQL](https://learn.microsoft.com/sql/connect/ado-net/sql/writing-secure-dynamic-sql). diff --git a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/references/quotename-null-and-order-by-clashes.md b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/references/quotename-null-and-order-by-clashes.md new file mode 100644 index 0000000..b39ced4 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/references/quotename-null-and-order-by-clashes.md @@ -0,0 +1,211 @@ +# QUOTENAME returning NULL, and the ORDER BY type clashes: the measured runs + +## Contents + +- [How this was measured](#how-this-was-measured) +- [QUOTENAME above 128 characters](#quotename-above-128-characters) +- [The silent no-op, end to end](#the-silent-no-op-end-to-end) +- [QUOTENAME with an embedded closing bracket](#quotename-with-an-embedded-closing-bracket) +- [CASE in ORDER BY mixing types](#case-in-order-by-mixing-types) +- [The safe pattern, with OFFSET FETCH](#the-safe-pattern-with-offset-fetch) +- [What is documented rather than measured](#what-is-documented-rather-than-measured) + +## How this was measured + +One engine, reachable over the network, reporting: + +```text +SERVERPROPERTY('EngineEdition') = 5 +SERVERPROPERTY('Edition') = SQL Azure +``` + +Date of the run: 2026-08-29. A dedicated database, `dq_injection`, carried a table: + +```sql +CREATE TABLE dbo.Grid (id INT, name NVARCHAR(50), amount DECIMAL(10,2), created DATE); +INSERT INTO dbo.Grid VALUES (1,'b',10.50,'2026-01-01'),(2,'a',20.25,'2026-02-01'),(3,'c',5.00,'2026-03-01'); +ALTER TABLE dbo.Grid ADD nickname NVARCHAR(50) NULL; +UPDATE dbo.Grid SET nickname = name; +``` + +## QUOTENAME above 128 characters + +```sql +DECLARE @n128 NVARCHAR(200) = REPLICATE(N'a', 128); +DECLARE @n129 NVARCHAR(200) = REPLICATE(N'a', 129); +SELECT LEN(@n128) AS len128, QUOTENAME(@n128) AS q128, LEN(QUOTENAME(@n128)) AS qlen128; +SELECT LEN(@n129) AS len129, QUOTENAME(@n129) AS q129, LEN(QUOTENAME(@n129)) AS qlen129; +``` + +Result: at exactly 128 characters, `QUOTENAME` returns the bracketed string (130 +characters once wrapped) with no complaint. At exactly 129 characters, `QUOTENAME` +returns `NULL`. The boundary is precise, not approximate, and it is `NULL`, not a +truncated or escaped value. + +## The silent no-op, end to end + +```sql +DECLARE @tbl NVARCHAR(400) = REPLICATE(N'x', 129); +DECLARE @sql NVARCHAR(MAX) = N'SELECT * FROM ' + QUOTENAME(@tbl); +PRINT 'sql value is: ' + ISNULL(@sql, N'<>'); +EXEC sp_executesql @sql; +PRINT 'executed with no error, @@ERROR=' + CAST(@@ERROR AS NVARCHAR(10)); +``` + +Output: + +```text +sql value is: <> +executed with no error, @@ERROR=0 +``` + +No rows, no error, no non-zero `@@ERROR`, exit code 0. `sp_executesql` was given a +`NULL` batch and treated it as nothing to do, not as a malformed call. + +This is not specific to `sp_executesql`. The same `NULL` batch handed to `EXEC(@sql)` +behaves identically: control reaches the next statement, `@@ERROR` is 0, and sqlcmd's +own process exit code is 0 in both cases. So the failure mode is "a `NULL` dynamic +batch executes as a no-op", not a quirk of one calling convention. Anywhere a +concatenated identifier can silently become `NULL` (a bad `QUOTENAME` call is the +common source, but plain string concatenation with a `NULL` variable does the same +thing), the statement built from it disappears without a trace at any severity level. + +## QUOTENAME with an embedded closing bracket + +```sql +SELECT QUOTENAME(N'my]col') AS escaped; +-- [my]]col] +``` + +`QUOTENAME` doubles an embedded `]` correctly, the standard T-SQL bracket-escaping +convention, so escaping itself is not the defect. The defect is exclusively the +128-character ceiling and its `NULL` return, which nothing downstream reports. + +## CASE in ORDER BY mixing types + +Four columns: `id INT`, `name NVARCHAR(50)`, `amount DECIMAL(10,2)`, `created DATE`. +Each test picks a column at runtime with `CASE @col WHEN 'x' THEN colx WHEN 'y' THEN +coly END`, the shape an agent reaches for to make `ORDER BY` accept a user-chosen +column without ever putting an identifier in a parameter. + +The first pass at this (below, "mixed at all") only asked whether a pairing fails. +Re-measured with the sort key varied across its own possible values, on a second +table, `dbo.T (i INT, d DATE, s NVARCHAR(50))` holding two rows, `(1, '2026-01-01', +N'Ann')` and `(2, '2026-02-01', N'Bob')`: the runtime pairings do not fail +unconditionally. They fail only for the sort key value that selects the +lower-precedence branch, and the exact same statement runs clean for the other +value. Presenting a runtime pairing as "fails" without both values is exactly the +setup where a reader tests the passing value, ships it, and only hits the failing +value in production. + +### int mixed with date, or decimal mixed with date: unconditional, compile time + +```sql +DECLARE @c CHAR(1) = N'a'; -- or 'b', or a value matching neither +SELECT * FROM dbo.T +ORDER BY CASE @c WHEN N'a' THEN i WHEN N'b' THEN d END; +``` + +`Msg 206, Level 16: Operand type clash: int is incompatible with date.` Fires for +every value of `@c`, including one that matches neither branch, because there is +no implicit conversion between `int` and `date` at all: the clash is detected +compiling the expression, before any row or any branch selection is evaluated. +`amount` (decimal) mixed with `created` (date) fails the same way: `Msg 206, +Operand type clash: decimal is incompatible with date.` + +This one is also not catchable in the same batch. Wrapping the statement in +`BEGIN TRY` / `END CATCH` does not run the `CATCH` block; the batch aborts at the +clash, before the surrounding `TRY` has a chance to take effect: + +```sql +BEGIN TRY + SELECT * FROM dbo.T ORDER BY CASE @c WHEN N'a' THEN i WHEN N'b' THEN d END; +END TRY +BEGIN CATCH + PRINT 'caught'; -- never reached +END CATCH +-- result: Msg 206 raised directly, nothing after it in the batch runs, no PRINT +``` + +### int or date mixed with a string column: only when the matched branch is the string + +```sql +-- branches: 'a' -> s (nvarchar), 'b' -> d (date) +DECLARE @c CHAR(1) = N'a'; -- selects s, the lower-precedence branch +ORDER BY CASE @c WHEN N'a' THEN s WHEN N'b' THEN d END; +``` + +| Branches | `@c` | Branch selected | Result | +|---|---|---|---| +| `'a' THEN s` (nvarchar), `'b' THEN d` (date) | `'b'` | `d`, the higher-precedence side | Runs clean, 2 rows | +| `'a' THEN s` (nvarchar), `'b' THEN d` (date) | `'a'` | `s`, the lower-precedence side | `Msg 241, Level 16: Conversion failed when converting date and/or time from character string.` | +| `'a' THEN i` (int), `'b' THEN s` (nvarchar) | `'a'` | `i`, the higher-precedence side | Runs clean, 2 rows | +| `'a' THEN i` (int), `'b' THEN s` (nvarchar) | `'b'` | `s`, the lower-precedence side | `Msg 245, Level 16: Conversion failed when converting the nvarchar value 'Ann' to data type int.` | +| Either pairing above | a value matching neither `'a'` nor `'b'`, e.g. `'x'` | neither; `CASE` falls through to implicit `ELSE NULL` | Runs clean, 2 rows, no conversion attempted | + +`date` and `int` both outrank `nvarchar` in type precedence, so the string branch +is the one coerced toward the winning type, and only the row values actually +reached by the **selected** branch are ever converted. When the selected branch is +already the winning type, nothing needs converting and the statement runs whether +or not the other, unreached branch would have failed. This is what makes the +runtime pairings a sharper trap than the compile-time one: the statement is +correct T-SQL, passes a test run against whichever sort key was tried, and fails +only when a caller picks the other one, in production, against a clause that has +not changed. + +### Same-typed and freely-convertible pairings never hit this + +| Branches mixed | Result | +|---|---| +| `name` (nvarchar), `nickname` (nvarchar) | Runs cleanly for every value of the sort key. Both branches are the same type. | +| `id` (int), `amount` (decimal) | Runs cleanly for every value of the sort key. Both are numeric types with a defined implicit conversion between them. | + +### The fix removes the value-dependence, not just the error + +One `CASE` **per candidate column**, each with a single type across its own +branches, has no mixed-type common type to coerce toward, at compile time or at +runtime. That is true whichever sort key is passed, so the fix is not "avoid the +failing value," it is "make there be no failing value." See +[the safe pattern below](#the-safe-pattern-with-offset-fetch). + +## The safe pattern, with OFFSET FETCH + +```sql +DECLARE @col SYSNAME = N'created', @dir CHAR(1) = N'A'; +SELECT * FROM dbo.Grid +ORDER BY + CASE WHEN @col = N'id' AND @dir = N'A' THEN id END ASC, + CASE WHEN @col = N'id' AND @dir = N'D' THEN id END DESC, + CASE WHEN @col = N'name' AND @dir = N'A' THEN name END ASC, + CASE WHEN @col = N'name' AND @dir = N'D' THEN name END DESC, + CASE WHEN @col = N'created' AND @dir = N'A' THEN created END ASC, + CASE WHEN @col = N'created' AND @dir = N'D' THEN created END DESC +OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY; +``` + +Ran cleanly, correctly sorted by `created`, three rows returned. This is the fix that +actually works: one `CASE` expression **per candidate column**, each with a single +type in its `THEN`, rather than one `CASE` with every candidate column as a branch. +The columns that do not match `@col` evaluate to `NULL` for every row and sort as +ties, so ordering by the chosen column is unaffected. `OFFSET`/`FETCH` composes with +this pattern without any further interaction; it does not need `ORDER BY` to name a +literal column, only a valid ordering expression, and this is one. + +A server-side allowlist (map the UI's sort key to a fixed, reviewed set of column +names before ever building SQL) and this per-column `CASE` pattern are the two +approaches that hold. Neither ever puts a column name where a parameter would go, and +neither depends on `QUOTENAME` at all, so the 128-character defect above does not +apply to either. + +## What is documented rather than measured + +**Always Encrypted and a literal predicate.** Microsoft Learn documents that a query +comparing an Always Encrypted column to a literal fails, because the literal is never +encrypted client-side and the comparison becomes a plaintext-to-ciphertext operand type +clash; only a parameter passed through a driver with `Column Encryption Setting=Enabled` +is encrypted before it reaches the server, so parameterisation is required for the +query to run at all, not merely safer. **This was not reproduced against the engine +used for the rest of this file**, because Always Encrypted needs a column master key +provisioned outside the database (a certificate store, Azure Key Vault, or an HSM), +which this run did not set up. Treat the QUOTENAME and CASE findings above as measured, +and this one as documented. diff --git a/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/skill.spec.jsonc new file mode 100644 index 0000000..521b41c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prevent-sql-injection/skill.spec.jsonc @@ -0,0 +1,196 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "prevent-sql-injection", + "domain": "security-and-safe-sql", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "A probe with no skill loaded found that an agent asked about SQL injection on Azure SQL Database already reaches for parameters and a server-side allowlist unprompted, and is already correct on ownership chaining, on Dynamic Data Masking not being a security boundary, and on which SQL Server escalation paths do not exist here, so a skill that repeats 'use parameterised queries' teaches nothing. What the same probe found wrong survives as the actual correction: QUOTENAME returns NULL for any identifier over 128 characters rather than truncating or erroring, so a dynamic statement concatenated from it becomes NULL and sp_executesql, or EXEC() of the same NULL batch, then runs it as a silent no-op with @@ERROR at 0 and no rows, which both probed model tiers got wrong in different ways (one said QUOTENAME fails to check the object exists, the other said it truncates); a single CASE expression choosing between columns of different types in a dynamic ORDER BY is coerced to one common type by precedence, and int mixed with date fails unconditionally at compile time (Msg 206, not even catchable in the same TRY/CATCH), but int or date mixed with a string column fails only when the sort key selects the string branch, the identical statement running clean when the sort key selects the other branch or matches neither (Msg 245 or Msg 241 only on the failing value), so it passes a test run against whichever sort key was tried and fails only when a caller picks the other one; and Always Encrypted rejects a literal predicate against an encrypted column outright, so on such a column parameterisation is mandatory for the query to run at all rather than a hardening choice, which corrected only the smaller probed model tier and was not reproduced against a live engine because it needs a column master key.", + "value_probe": { + "gate": "The answer traces the empty grid to a dynamic statement that became nothing before it was executed, and the conversion failure to one expression forced into a single result type by precedence, and it rebuilds both so neither can recur. Restating the case for parameters and an allowlist, which this code already has, does not reach it.", + "task": "This procedure on Azure SQL Database drives a grid that lets the caller choose which table to read and which column to sort by. It has been in production for about a year.\n\n```sql\nCREATE PROCEDURE dbo.get_grid @table sysname, @sort nvarchar(50)\nAS\nDECLARE @sql nvarchar(max) =\n N'SELECT TOP (100) * FROM ' + QUOTENAME(@table) +\n N' ORDER BY CASE @s WHEN ''name'' THEN name ELSE created_date END';\nEXEC sp_executesql @sql, N'@s nvarchar(50)', @s = @sort;\n```\n\nThe table name is checked against a list held in sys.tables before we ever get here, so the usual question is not what I am asking about. I have two support tickets I cannot explain.\n\nOne tenant generates very long table names and their grid comes back completely empty. No error, no rows, nothing in our logs, and the error handler wrapped around the call never fires. It started only after their generated names got longer.\n\nAnd QA reports 'Conversion failed when converting date and/or time from character string' when they sort by name. Sorting by anything else works, and it worked in every test we ran before release.\n\nReview this and tell me what is actually wrong and how to fix it.", + "states": [ + "The identifier helper returns nothing at all once the name passed to it exceeds the identifier length limit, rather than shortening it or failing loudly.", + "Concatenating anything onto that nothing makes the whole statement nothing, so the execution does no work and reports no error, which is why the handler never fires.", + "A CASE expression has one result type decided by data type precedence, so every branch is converted into that one type.", + "The conversion fails only for the values that select the branch needing the conversion, which is why the statement passed every test that used the other sort key.", + "The sort has to be rebuilt so no branch depends on converting one column type into another." + ], + "refutes": [ + "The identifier helper shortens a name that is too long, or raises an error on one, so an over-length table name could not have produced a silent empty result.", + "The identifier helper checks that the object exists, so an unrecognised or malformed table name is rejected before the dynamic statement is executed.", + "Executing a statement variable that holds nothing raises an error, so the empty grid must be caused by the data, the permissions or the tenant's own filtering.", + "Each branch of a CASE expression keeps its own data type, so mixing a text column and a date column in one sort expression is valid.", + "Both tickets are symptoms of an injection weakness, and parameterising the statement or validating the table name will fix them." + ] + }, + "posture": ["read", "write"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "rls-multi-tenant", + "t-sql-correctness", + "design-azure-sql-schema", + "entra-id-auth", + "connect-from-dotnet", + "connect-from-python", + "connect-from-typescript-and-node" + ], + "validation": { + "target": "both", + "setup": [ + "a table with columns of at least three different data types, including one string and one date or datetime column", + "the ability to run a batch that builds an identifier longer than 128 characters" + ], + "assert": [ + "a generated dynamic statement that builds an object name never relies on QUOTENAME alone above 128 characters without checking the result for NULL before executing it", + "a generated dynamic ORDER BY that lets the caller choose the sort column uses one CASE expression per candidate column rather than one CASE with every candidate column as a branch, or uses a server-side allowlist instead of CASE entirely", + "no generated answer claims QUOTENAME truncates a long identifier or merely fails to check the object exists; it returns NULL", + "an answer touching an Always Encrypted column states that a literal predicate against it fails outright, not merely that parameters are the safer choice" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_inj_users (id int NOT NULL PRIMARY KEY, name nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_inj_users (id, name) VALUES (1, N'ann'), (2, N'bob');\nCREATE TABLE dbo.probe_inj_concat (matched int NOT NULL);\nDECLARE @input nvarchar(100) = N'ann'' OR 1=1--';\nDECLARE @parameterised int;\nEXEC sp_executesql N'SELECT @out = COUNT(*) FROM dbo.probe_inj_users WHERE name = @n', N'@n nvarchar(100), @out int OUTPUT', @n = @input, @out = @parameterised OUTPUT;\nDECLARE @sql nvarchar(max) = N'INSERT INTO dbo.probe_inj_concat SELECT COUNT(*) FROM dbo.probe_inj_users WHERE name = N''' + @input + N'''';\nEXEC (@sql);\nSELECT @parameterised AS parameterised, (SELECT matched FROM dbo.probe_inj_concat) AS concatenated;", + "expect": { + "rows": [ + [ + 0, + 2 + ] + ] + }, + "why": "Section 1 made executable, and the claim the whole skill rests on. One string reaches both statements. Bound to a typed sp_executesql parameter it is a value that matches no row, so 0. Concatenated into the text of the same query it is code, the trailing -- comments out the closing quote, OR 1=1 survives, and every row comes back, so 2. Zero against two from one input is the difference between a value and a statement, measured rather than asserted." + }, + { + "sql": "SELECT LEN(QUOTENAME(REPLICATE(N'a', 128))) AS len_at_128,\n IIF(QUOTENAME(REPLICATE(N'a', 129)) IS NULL, 1, 0) AS null_at_129,\n IIF(QUOTENAME(N'tbl', N'#') IS NULL, 1, 0) AS null_bad_quote,\n IIF(QUOTENAME(N'my]col') = N'[my]]col]', 1, 0) AS bracket_doubled;", + "expect": { + "rows": [ + [ + 130, + 1, + 1, + 1 + ] + ] + }, + "why": "The 128 boundary is precise and it is a NULL, not a truncation. Microsoft Learn states that character_string is sysname, limited to 128 characters, and that inputs greater than 128 characters return NULL, and that an unacceptable quote character returns NULL. Measured 2026-08-29: 128 characters come back bracketed at length 130 and 129 come back NULL. The fourth column pins the half an agent gets right, so a future failure cannot be blamed on escaping." + }, + { + "sql": "DECLARE @sql nvarchar(max) = N'SELECT 1 AS x FROM ' + QUOTENAME(REPLICATE(N'x', 129));\nEXEC sp_executesql @sql;\nDECLARE @err_sp int = @@ERROR;\nEXEC (@sql);\nDECLARE @err_exec int = @@ERROR;\nSELECT IIF(@sql IS NULL, 1, 0) AS batch_was_null, @err_sp AS err_after_sp_executesql, @err_exec AS err_after_exec;", + "expect": { + "rows": [ + [ + 1, + 0, + 0 + ] + ] + }, + "why": "The silent no-op, asserted through both calling conventions so the claim cannot be pinned on one of them. A NULL from QUOTENAME makes the whole concatenated batch NULL, and handing that NULL to sp_executesql and to EXEC() both leave @@ERROR at 0 with nothing executed. Measured 2026-08-29. This probe is the reason the skill tells a reader to check the NULL rather than the error, and a failure here would mean the engine started reporting it." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_intdate (i int NOT NULL, d date NOT NULL);\nINSERT INTO dbo.probe_inj_intdate (i, d) VALUES (1, '2026-01-01');\nDECLARE @col sysname = N'matches_no_branch';\nSELECT i FROM dbo.probe_inj_intdate ORDER BY CASE WHEN @col = N'i' THEN i WHEN @col = N'd' THEN d END;", + "expect": { + "error": 206 + }, + "why": "int and date have no implicit conversion, so one CASE spanning them is refused while the expression is compiled, before any branch is chosen. The sort key here deliberately matches NEITHER branch, which is what makes this unconditional rather than value-dependent. Measured 2026-08-29: Msg 206, operand type clash, int is incompatible with date." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_prec (i int NOT NULL, s nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_inj_prec (i, s) VALUES (2, N'bob'), (1, N'ann');\nDECLARE @col sysname = N's';\nSELECT i FROM dbo.probe_inj_prec ORDER BY CASE WHEN @col = N'i' THEN i WHEN @col = N's' THEN s END;", + "expect": { + "error": 245 + }, + "why": "The trap that survives testing. int outranks nvarchar by type precedence, so the common type is int and the string branch is the one converted, but only when the sort key selects it. Measured 2026-08-29: Msg 245, conversion failed when converting the nvarchar value to data type int. Paired with the next probe, which runs the IDENTICAL statement and passes." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_prec_ok (i int NOT NULL, s nvarchar(50) NOT NULL);\nINSERT INTO dbo.probe_inj_prec_ok (i, s) VALUES (2, N'bob'), (1, N'ann');\nDECLARE @col sysname = N'i';\nSELECT i FROM dbo.probe_inj_prec_ok ORDER BY CASE WHEN @col = N'i' THEN i WHEN @col = N's' THEN s END;", + "expect": { + "rows": [ + [ + 1 + ], + [ + 2 + ] + ] + }, + "why": "The other half of the pair, and the reason the skill calls this sharper than mixing types fails. Byte for byte the same ORDER BY as the probe above, with only the sort key changed to the higher-precedence branch, and it returns both rows in order. An author who tested this column alone would ship the previous probe's failure. Measured 2026-08-29." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_safe (id int NOT NULL, name nvarchar(50) NOT NULL, created date NOT NULL);\nINSERT INTO dbo.probe_inj_safe (id, name, created) VALUES (2, N'bob', '2026-02-01'), (1, N'ann', '2026-01-01');\nDECLARE @col sysname = N'created';\nSELECT id FROM dbo.probe_inj_safe\nORDER BY CASE WHEN @col = N'id' THEN id END,\n CASE WHEN @col = N'name' THEN name END,\n CASE WHEN @col = N'created' THEN created END\nOFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY;", + "expect": { + "rows": [ + [ + 1 + ], + [ + 2 + ] + ] + }, + "why": "The remedy the skill actually prescribes, run on the sort key that breaks the single-CASE form. One CASE per candidate column gives each expression exactly one type, so there is no common type to coerce toward and no failing branch left. Sorting by created returns id 1 before id 2, which also shows the non-matching CASE expressions evaluating to NULL and tying rather than reordering, and that it composes with OFFSET and FETCH." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_ledger (id int NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_inj_ledger (id) VALUES (1);\nCREATE USER probe_inj_reader WITHOUT LOGIN;\nEXEC sys.sp_executesql N'CREATE PROCEDURE dbo.probe_inj_dynamic AS EXEC sp_executesql N''SELECT COUNT(*) AS n FROM dbo.probe_inj_ledger'';';\nGO\nGRANT EXECUTE ON dbo.probe_inj_dynamic TO probe_inj_reader;\nEXECUTE AS USER = 'probe_inj_reader';\nEXEC dbo.probe_inj_dynamic;", + "expect": { + "error": 229 + }, + "why": "Section 4 made executable. Microsoft Learn states that executing dynamically created SQL in procedural code breaks the ownership chain, so the engine checks the caller against every object the dynamic statement touches. The caller holds EXECUTE on the procedure and nothing on the table, and a fully parameterised dynamic statement still fails: Msg 229, the SELECT permission was denied on the object. This is the measurement behind the claim that parameterising does not restore the chain." + }, + { + "sql": "CREATE TABLE dbo.probe_inj_ledger_owner (id int NOT NULL PRIMARY KEY);\nINSERT INTO dbo.probe_inj_ledger_owner (id) VALUES (1);\nCREATE USER probe_inj_reader_owner WITHOUT LOGIN;\nEXEC sys.sp_executesql N'CREATE PROCEDURE dbo.probe_inj_as_owner WITH EXECUTE AS OWNER AS EXEC sp_executesql N''SELECT COUNT(*) AS n, CURRENT_USER AS running_as FROM dbo.probe_inj_ledger_owner'';';\nGO\nGRANT EXECUTE ON dbo.probe_inj_as_owner TO probe_inj_reader_owner;\nEXECUTE AS USER = 'probe_inj_reader_owner';\nEXEC dbo.probe_inj_as_owner;", + "expect": { + "rows": [ + [ + 1, + "dbo" + ] + ] + }, + "why": "The remedy, and its cost, in one result. The identical procedure with WITH EXECUTE AS OWNER returns the row the previous probe was refused, and CURRENT_USER inside it reports dbo rather than the caller. Microsoft Learn states that some functions returning the identity of the user return the user named in the EXECUTE AS clause, not the original caller, and that this can break applications relying on row level security. The second column is why the skill sends a reader to rls-multi-tenant before adding this clause." + } + , + { + "sql": "CREATE TABLE dbo.probe_psi_id (i int, d date);\nGO\nSELECT CASE WHEN 1 = 1 THEN i ELSE d END AS x FROM dbo.probe_psi_id;", + "expect": { + "error": 206 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A mixed type CASE over int and date is an operand type clash at compile time, the first of the three numbers that mean a CASE is doing the type coercion a developer thought was safe. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_psi_sd (d date, s nvarchar(20), pick int);\nINSERT INTO dbo.probe_psi_sd VALUES ('2026-01-01', N'abc', 1);\nGO\nSELECT CASE WHEN pick = 1 THEN s ELSE d END AS x FROM dbo.probe_psi_sd;", + "expect": { + "error": 241 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. String against date resolves to the date side and fails converting, which is a run time failure on the data and not a compile time one. The row is inserted here so the conversion is actually attempted. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_psi_is (i int, s nvarchar(20), pick int);\nINSERT INTO dbo.probe_psi_is VALUES (1, N'abc', 0);\nGO\nSELECT CASE WHEN pick = 1 THEN i ELSE s END AS x FROM dbo.probe_psi_is;", + "expect": { + "error": 245 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Int against string resolves to int and fails on the value, the third number. The skill's rule is that any of 206, 241 or 245 means a mixed type CASE, and these three probes are that rule made executable. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Why did my QUOTENAME-based dynamic SQL run successfully but return nothing, when the table name is unusually long?", + "Write a dynamic ORDER BY for Azure SQL Database that lets the caller sort by any column, including a date column, without hitting an operand type clash", + "Is it safe to build my table name with QUOTENAME before running it with sp_executesql?", + "My sort-by-column endpoint throws Msg 241 the first time someone asks to sort by the date column, but every other column works" + ], + "implicit": [ + "sp_executesql completed with no error and no rows, and the object name it built is over 100 characters", + "our generic ORDER BY CASE worked in every test until QA sorted by created_date instead of name", + "Conversion failed when converting date and/or time from character string, and the query has a CASE in its ORDER BY", + "customers with very long generated names get zero rows back with no error at all, and it only started once names got long" + ], + "negative": [ + "Set up row-level security so one tenant can never see another tenant's rows", + "Escape the user's comment before we render it back into the HTML page", + "My Postgres app uses $1 placeholders, what is the equivalent when I port these queries", + "Rotate the database password our app connects with and keep it out of source control", + "Clean up the CSV before we load it into the table", + "Grant the reporting team read access to just two tables and nothing else" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/prisma-azure-sql b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql deleted file mode 120000 index d90e52e..0000000 --- a/plugins/microsoft-sql-vscode/skills/prisma-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/prisma-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/SKILL.md new file mode 100644 index 0000000..369ccd5 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/SKILL.md @@ -0,0 +1,293 @@ +--- +name: prisma-azure-sql +description: >- + Uses Prisma ORM against Azure SQL Database on JavaScript and TypeScript, inside the connector's + real limits: no Json type, no enums, no scalar lists, a default string length that quietly + breaks keys, a connection URL that moved out of the schema file in Prisma 7, and a migration + workflow that succeeds locally and is refused in the cloud. Use when a user says "Prisma with + Azure SQL", "prisma migrate dev", "prisma db push", "schema.prisma", "prisma.config.ts", + "driver adapter", or pastes "P3020", "the automatic creation of shadow databases is disabled", + "the current connector does not support the Json type", or "the datasource property url is no + longer supported". Also use when Prisma migrations work locally and fail against Azure. Covers + schema, type mapping, migrations and identity for Prisma only. Drivers and pooling are + connect-from-typescript-and-node, retry connect-to-azure-sql. +--- + +# Prisma on Azure SQL Database + +Prisma's support for this database is real, narrower than its PostgreSQL support, and it moved in +Prisma 7. A schema written from PostgreSQL habits fails validation, and a migration workflow proved +against a local database is refused in the cloud. + +Measured on 2026-09-03 against `prisma` 7.9.1 and 7.10.0, `@prisma/adapter-mssql` 7.9.1 and `prisma` +8.0.0-rc.12, all installed from the package index. The P3020 result below was measured on 2026-08-27 +against an Azure SQL Database engine reporting `EngineEdition` 5. + +The datasource provider is the literal string `sqlserver`. **That is a connector identifier, not the +name of the product**, whose limits are not SQL Server's: everywhere else it is Azure SQL Database. + +## Pin the version, because the newest has no connector at all + +```bash +npm view prisma dist-tags # latest: 8.0.0-rc.12 prev: 7.10.0 +npm install prisma@7 @prisma/client@7 @prisma/adapter-mssql@7 +``` + +`latest` is a release candidate for the next major, so a bare `npm install prisma` installs it. The +string `sqlserver` appears nowhere in that package, only `postgresql`, and its command surface is a +different product: no `validate`, no `db push`, no `migrate`, and `migrate diff` replaced by a +`migration` group. Pin to 7 and keep `prisma` and `@prisma/client` equal. + +## The connection URL is JDBC style, not a URI + +Every other Prisma connector takes a URI. This one does not: + +```bash +# rejected: P1013, "Conversion error: invalid digit found in string in database URL" +sqlserver://user:password@.database.windows.net:1433/appdb + +# accepted +sqlserver://.database.windows.net:1433;database=appdb;user=app;password=;encrypt=true +``` + +The error blames a digit and never mentions the shape, so it reads as a typo. An ADO.NET style +`Server=...;Database=...` string is rejected too, with `must start with the protocol sqlserver://`. + +## Prisma 7 moved the URL out of the schema file + +A `url` inside the `datasource` block now fails `prisma validate` with `P1012`, pointing at the line: + +``` +The datasource property `url` is no longer supported in schema files. +``` + +The rest of that message sends the URL to `prisma.config.ts` and the adapter to the client: + +```ts +// prisma.config.ts +import 'dotenv/config' // nothing else loads .env +import { defineConfig, env } from 'prisma/config' + +export default defineConfig({ + schema: 'prisma/schema.prisma', + datasource: { + url: env('DATABASE_URL'), + shadowDatabaseUrl: env('SHADOW_DATABASE_URL'), + }, +}) +``` + +`env()` is strict and fails the whole command, including `prisma validate`, which needs no database: +`PrismaConfigEnvError: Cannot resolve environment variable: DATABASE_URL`. The `dotenv/config` import +is what stops that; the configuration file loads no `.env` on its own. + +The client needs a driver adapter as well. `@prisma/adapter-mssql` wraps `mssql` ^12.2.0, the Node +driver `connect-from-typescript-and-node` owns. + +## What the connector refuses, before any database is involved + +```bash +./node_modules/.bin/prisma validate +``` + +Three field kinds are rejected outright, all three reproduced on 7.9.1: + +``` +error: Field `payload` in model `Thing` can't be of type Json. The current connector does not support the Json type. +error: Field "tags" in model "Thing" can't be a list. The current connector does not support lists of primitive types. +error: Error validating: You defined the enum `Status`. But the current connector does not support enums. +``` + +Instead: **JSON** is a string column with the shape enforced in application code, since the +database's own JSON handling needs a raw query; an **enum** is a lookup table with a relation, the +form that survives contact with reporting; a **scalar list** is a child table. + +## The default string length, and the failure it defers + +Read the DDL Prisma will emit before it runs. This needs a parseable URL and opens no connection: + +```bash +./node_modules/.bin/prisma migrate diff --from-empty --to-schema=prisma/schema.prisma --script +``` + +For `id String @id` and `email String @unique` with no native type, that prints: + +```sql +[id] NVARCHAR(1000) NOT NULL, +[email] NVARCHAR(1000) NOT NULL, +CONSTRAINT [Account_pkey] PRIMARY KEY CLUSTERED ([id]), +CONSTRAINT [Account_email_key] UNIQUE NONCLUSTERED ([email]) +``` + +1000 Unicode characters is **2000 bytes**, against index key limits of **1700 bytes** nonclustered +and **900 bytes** clustered. Nothing fails at migration time. The engine emits `Msg 1945` at +**severity 10**, which succeeds: + +``` +Warning! The maximum key length for a nonclustered index is 1700 bytes. +The index 'Account_email_key' has maximum length of 2000 bytes. +For some combination of large values, the insert/update operation will fail. +``` + +Severity 10 sets no non-zero exit, so `sqlcmd -b` alone reports a clean migration and the pipeline +goes green. Without `-m-1` the message prints no `Msg` number at all, and on go-sqlcmd it prints no +number with `-m-1` either, which the check section below explains. The bill arrives later, on a +genuinely long value: + +``` +Msg 1946, Level 16, State 3 +Operation failed. The index entry of length 1800 bytes for the index 'Account_email_key' +exceeds the maximum length of 1700 bytes for nonclustered indexes. +``` + +`String @id` is worse: a clustered primary key against the 900 byte limit, again with a warning only. +**So size every string that carries a key, a unique constraint or a foreign key.** + +```prisma +model Account { + id String @id @db.NVarChar(64) + email String @unique @db.NVarChar(320) + bio String? @db.NVarChar(Max) +} +``` + +Re-running the diff emits `NVARCHAR(64)`, `NVARCHAR(320)` and `NVARCHAR(max)`. 450 characters is the +widest value safe in every index position; anything outside an index can be longer, and unbounded +text belongs in `NVarChar(Max)` rather than a large fixed default. + +One more PostgreSQL habit, same area: a **nullable unique column accepts one null row only**. The +second fails `Msg 2627`, `The duplicate key value is ()`. A model with several optional unique +fields works on other engines and does not work here. + +That number depends on what Prisma built. `@unique` and `@@unique` emit +`ADD CONSTRAINT ... UNIQUE NONCLUSTERED`, a **constraint**, on both the create and the alter path, +though Prisma labels the step `-- CreateIndex` in the migration. A constraint violation is +`Msg 2627`; a hand-written `CREATE UNIQUE INDEX` raises `Msg 2601`. Match the number to the object +or you search for the wrong one. Key design in general is `design-azure-sql-schema`. + +## Migrations, and the one that will not run in the cloud + +`prisma migrate dev` needs a **shadow database**: a second temporary database it creates, replays +history into, inspects and drops, so it can detect drift. `prisma migrate deploy` does not use one. + +Against Azure SQL Database, Prisma will not create it and stops with a dedicated error: + +``` +P3020 +The automatic creation of shadow databases is disabled on Azure SQL. +Please set up a shadow database using the `shadowDatabaseUrl` datasource attribute. +``` + +Three things follow. + +1. **A local success proves nothing, and this is not a permission problem.** Against the local Azure + SQL Database container the shadow database is created and dropped and `migrate dev` works. The + same schema and the same administrative login return P3020 in the cloud, so granting rights fixes + nothing, and a pipeline is where this is found. +2. **The error names a place that no longer exists.** "datasource attribute" was the Prisma 6 + location. In Prisma 7 `shadowDatabaseUrl` sits in the configuration file beside `url`. +3. **The fix is a second database, not a grant.** Provision another database on the same logical + server and point `SHADOW_DATABASE_URL` at it; Prisma refuses if the two match, with `The shadow + database you configured appears to be the same as the main database.` Creating one needs the + server administrator, the Entra administrator or the database manager role, which an application + login is not: that is `provision-azure-sql-db`. + +| Situation | Command | Shadow database | +|---|---|---| +| Iterating against the container | `prisma migrate dev` | Created and dropped for you | +| Iterating against a cloud database | `prisma migrate dev` | You provide it, once | +| Prototyping with no migration history | `prisma db push` | Not used | +| Deploying | `prisma migrate deploy` | Not used | + +Never run `migrate dev` against a database holding real data: it is the command that resets. Whether +a change is safe for a live database at all is `schema-migrations-safely`. + +## Getting an identity to the client + +The adapter supports Microsoft Entra ID and the Prisma documentation does not mention it, so an agent +working from the docs alone concludes it is impossible. Read out of the installed 7.9.1 package, the +connection string accepts `authentication=DefaultAzureCredential`, +`authentication=ActiveDirectoryManagedIdentity` with an optional client id, and +`authentication=ActiveDirectoryServicePrincipal`; the object form is +`authentication: { type: 'azure-active-directory-default' }`. + +**The command line is a separate path.** Migrations connect through the configuration file's URL, +whose documented arguments are a user and a password, so plan for them to run under a login even +where the application is passwordless. The identity still needs a database principal, which is +`entra-id-auth`. + +## Check it worked + +**The schema is one the connector accepts.** With `DATABASE_URL` exported: + +```bash +./node_modules/.bin/prisma validate +``` + +Expect exactly `The schema at prisma/schema.prisma is valid 🚀`. Any `P1012` block is a Json field, +an enum or a scalar list; a `PrismaConfigEnvError` means the `dotenv/config` import is missing. + +**No column is heading for Msg 1946.** Read the DDL before it runs: + +```bash +./node_modules/.bin/prisma migrate diff --from-empty --to-schema=prisma/schema.prisma --script | grep -c 'NVARCHAR(1000)' +``` + +Expect `0`. Each match is one unsized `String`, and any of them in a key, unique constraint or +foreign key is a migration that warns and an insert that fails later. + +**Nothing already migrated is over the limit.** Against the migrated database: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 -Q \ + "SELECT t.name AS tbl, i.name AS ix, i.type_desc, SUM(c.max_length) AS key_bytes + FROM sys.indexes AS i + JOIN sys.index_columns AS ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id + AND ic.is_included_column = 0 + JOIN sys.columns AS c ON c.object_id = ic.object_id AND c.column_id = ic.column_id + JOIN sys.tables AS t ON t.object_id = i.object_id + GROUP BY t.name, i.name, i.type_desc + HAVING SUM(c.max_length) > CASE WHEN i.type_desc = 'CLUSTERED' THEN 900 ELSE 1700 END;" +``` + +Expect zero rows. Each row is an index over its limit, `key_bytes` 2000 being the unsized default. +Keep `-m-1`: without it the create-time warning carries no `Msg` number, which is how these ship. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build the 1945 warning `prisma migrate deploy` provokes stays +unnumbered and the migration still reads as clean. Run the migration through the ODBC build when the +number is what you are grepping for. `build-app-on-azure-sql` tells the two builds apart in one +table. + +## Do not + +- Do not answer a `Json`, `enum` or scalar-list validation error with a raw query. Model it as a + column or a table. +- Do not read a green local migration as evidence that the cloud will accept it, and do not answer + P3020 with a permission grant or by pointing `shadowDatabaseUrl` at the database holding your data. +- Do not skip certificate validation, and do not write a retry loop here. Encryption and + transient-fault policy are one policy for every stack: `connect-to-azure-sql`. + +## References + +- [Prisma SQL Server connector](https://www.prisma.io/docs/orm/overview/databases/sql-server): the + type mapping table and every connection URL argument with its default. Fetch it before adding an + argument to the URL above. +- [Shadow database](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/shadow-database): + what `migrate dev` does with it, and the cloud-hosted setup. Read it before configuring one. +- [Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference): the definitive + text for P3020, P1012 and P1013. Read it when a P-code appears. +- [CREATE INDEX](https://learn.microsoft.com/sql/t-sql/statements/create-index-transact-sql): the + 900 and 1700 byte key limits in the engine's own words. Read it to confirm the rule, not the text. +- [Database Engine events and errors 1000 to 1999](https://learn.microsoft.com/sql/relational-databases/errors-events/database-engine-events-and-errors-1000-to-1999): + 1945 at severity 10 and 1946 at severity 16. Read it before asserting either number. +- `connect-from-typescript-and-node`: the `mssql` driver the adapter wraps. Open it before + configuring the adapter's pool, which is that driver's. +- `connect-to-azure-sql`: encryption, retry, transient faults. Open it the moment one appears. +- `design-azure-sql-schema`: key length, sizing, collation. Open it before the first migration, not + after the first `Msg 1946`. +- `schema-migrations-safely`: the doctrine Prisma Migrate inherits, including why a pipeline reports + success on a migration that only warned. +- `provision-azure-sql-db`: creating the second database P3020 asks for. Open it on that error. diff --git a/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..65a9ca6 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/prisma-azure-sql/skill.spec.jsonc @@ -0,0 +1,131 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "prisma-azure-sql", + "domain": "schema-and-orm", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize"], + "correction": "Agents author a Prisma schema for Azure SQL Database as though it were the PostgreSQL connector, reaching for Json, enum and scalar list fields the connector rejects outright and leaving String at its default 1000 character Unicode mapping, which is 2000 bytes and so over the index key limit, so a unique constraint migrates with nothing but a warning and fails much later on a real insert; and having watched prisma migrate dev succeed against a local database they assume it will work in the cloud, when Prisma refuses to create a shadow database on Azure SQL Database at all and stops with a dedicated error.", + "value_probe": { + "gate": "The answer keeps the reader on a release that actually contains the connector for this database, and gives a migration workflow that works against the cloud database rather than the one that only works against a local one.", + "task": "I am starting a TypeScript service against Azure SQL Database and I want the schema and the migrations handled by the same ORM we already use on our PostgreSQL projects. I need a model with a JSON payload column, a status field with a fixed set of values, a list of tags, and an email column that has to be unique. Give me the install commands, the schema file and the migration workflow to use, and tell me anything that will work on my machine and then not work against the cloud database.", + "states": [ + "The version installed has to be pinned, because the tag that installs by default is a release candidate for the next major.", + "The default string mapping is wide enough to exceed the index key limit, and the failure arrives later at insert time.", + "The cloud database refuses the automatic creation of the temporary database the development migration command needs.", + "A JSON payload becomes a string column, a fixed set of values becomes a lookup table, and a list becomes a child table." + ], + "refutes": [ + "Installing the ORM without pinning a version gives a release that supports this database.", + "A JSON column type, an enum and a list of scalar values are all available on this connector.", + "The development migration command that works against a local database works the same way against Azure SQL Database.", + "A string field left at its default length is safe to put a unique constraint on.", + "The connection URL for this connector is an ordinary URI like the other connectors take." + ] + }, + "posture": ["read", "write", "execute", "provision"], + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-from-typescript-and-node", + "connect-to-azure-sql", + "entra-id-auth", + "design-azure-sql-schema", + "schema-migrations-safely", + "ef-core-azure-sql", + "sqlalchemy-azure-sql", + "build-app-on-azure-sql", + "provision-azure-sql-db" + ], + "validation": { + "target": "both", + "setup": [ + "a pinned Prisma 7 release, since the connector does not exist in Prisma 8", + "a reachable Azure SQL database, or the local container", + "for migrate dev against the cloud, a second empty database to act as the shadow database" + ], + "assert": [ + "the schema declares no Json field, no enum block and no scalar list", + "every String column that carries a key, a unique constraint or a foreign key declares an explicit length of 450 characters or fewer", + "the connection URL is in the configuration file, not in the datasource block", + "migrate dev against a cloud database has a shadowDatabaseUrl pointing at a different database", + "deployments run migrate deploy, which uses no shadow database", + "no credential appears in the schema, the configuration file or the client construction" + ], + "probes": [ + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && npm install --no-audit --no-fund --silent prisma@7 >/dev/null 2>&1 && printf 'datasource db {\\n provider = \"sqlserver\"\\n}\\n\\nmodel Account {\\n id String @id\\n email String @unique\\n}\\n' > schema.prisma && printf \"import { defineConfig, env } from 'prisma/config'\\nexport default defineConfig({ schema: 'schema.prisma', datasource: { url: env('DATABASE_URL') } })\\n\" > prisma.config.ts && DATABASE_URL=\"sqlserver://localhost:${PROBE_PORT};database=${PROBE_DB};user=sa;password=x\" ./node_modules/.bin/prisma migrate diff --from-empty --to-schema=schema.prisma --script 2>/dev/null | grep -q 'NVARCHAR(1000)'", + "expect": { "succeeds": true }, + "why": "The central claim of the skill, made executable with no database involved: an unsized Prisma String maps to NVARCHAR(1000), which is 2000 bytes and over both index key limits. migrate diff --from-empty --to-schema needs a parseable datasource URL and opens no connection, which is why PROBE_PORT and PROBE_DB only have to make the URL well formed. Measured 2026-09-03 on the authoring host: exit 0 on prisma 7.9.1 and on 7.10.0, printing NVARCHAR(1000) for both the @id and the @unique column, PRIMARY KEY CLUSTERED on the first and UNIQUE NONCLUSTERED on the second. A red means the default mapping moved and the whole sizing section has to be re-measured." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && npm install --no-audit --no-fund --silent prisma@7 >/dev/null 2>&1 && printf 'datasource db {\\n provider = \"sqlserver\"\\n}\\n\\nmodel Thing {\\n id Int @id\\n payload Json\\n}\\n' > schema.prisma && printf \"import { defineConfig, env } from 'prisma/config'\\nexport default defineConfig({ schema: 'schema.prisma', datasource: { url: env('DATABASE_URL') } })\\n\" > prisma.config.ts && DATABASE_URL=\"sqlserver://localhost:${PROBE_PORT};database=${PROBE_DB};user=sa;password=x\" ./node_modules/.bin/prisma validate 2>&1 | grep -q 'does not support the Json type'", + "expect": { "succeeds": true }, + "why": "The connector's refusal list, asserted rather than described. grep exits 0 only when prisma validate actually prints the rejection, so this is a probe of the tool's behaviour and not of the skill's prose. Measured 2026-09-03 on the authoring host: exit 0, message text 'Field `payload` in model `Thing` can't be of type Json. The current connector does not support the Json type.' The enum and scalar-list refusals were reproduced in the same run and share this code path, so one probe covers the claim without paying for three installs." + }, + { + "where": "host", + "cmd": "d=$(mktemp -d) && cd \"$d\" && npm install --no-audit --no-fund --silent prisma@7 >/dev/null 2>&1 && printf 'datasource db {\\n provider = \"sqlserver\"\\n}\\n\\nmodel Thing {\\n id Int @id\\n}\\n' > schema.prisma && printf \"import { defineConfig, env } from 'prisma/config'\\nexport default defineConfig({ schema: 'schema.prisma', datasource: { url: env('DATABASE_URL') } })\\n\" > prisma.config.ts && DATABASE_URL=\"sqlserver://sa:x@localhost:${PROBE_PORT}/${PROBE_DB}\" ./node_modules/.bin/prisma migrate diff --from-config-datasource --to-empty --script 2>&1 | grep -q 'P1013'", + "expect": { "succeeds": true }, + "why": "The connection URL shape, which an earlier validation run recorded only as 'a Prisma sqlserver:// URL was rejected' without saying which shape. This settles it: the URI form sqlserver://user:password@host:port/database is refused with P1013 before any connection is attempted, because the parser reads 1433/appdb as a port and reports 'Conversion error: invalid digit found in string in database URL'. Measured 2026-09-03 on the authoring host: exit 0. The accepted form is the JDBC style sqlserver://HOST:PORT;database=..., which reached P1001 against a dead localhost and reached a real Azure SQL Database gateway when pointed at one." + }, + { + "sql": "CREATE TABLE dbo.probe_prisma_wide (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, email nvarchar(1000) NOT NULL);\nGO\nCREATE UNIQUE NONCLUSTERED INDEX probe_prisma_wide_email_key ON dbo.probe_prisma_wide (email);", + "expect": { "warning": 1945 }, + "why": "The deferred failure, caught at the moment it is deferred. nvarchar(1000) is 2000 bytes against the 1700 byte nonclustered limit. The fixture uses a standalone unique index while Prisma itself emits a UNIQUE constraint; Learn's Unique constraints and check constraints page states the engine creates a unique index to enforce the constraint, so the key limit and this warning are identical either way, and the index form is kept here because it is the form that was measured green. Microsoft Learn lists 1945 at severity 10 with the text 'Warning! The maximum key length for a %S_MSG index is %d bytes ... For some combination of large values, the insert/update operation will fail' (https://learn.microsoft.com/sql/relational-databases/errors-events/database-engine-events-and-errors-1000-to-1999), checked 2026-09-03. The statement succeeds, which is the entire problem: without sqlcmd -m-1 the message carries no Msg number and -b sets no non-zero exit below severity 11, so a pipeline goes green. An int primary key is used here so exactly one warning fires; Prisma's own String @id adds a second against the 900 byte clustered limit." + }, + { + "sql": "CREATE TABLE dbo.probe_prisma_insert (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, email nvarchar(1000) NOT NULL);\nGO\nCREATE UNIQUE NONCLUSTERED INDEX probe_prisma_insert_email_key ON dbo.probe_prisma_insert (email);\nGO\nINSERT INTO dbo.probe_prisma_insert (email) VALUES (REPLICATE(CAST(N'x' AS nvarchar(max)), 900));", + "expect": { "error": 1946 }, + "why": "The bill for the warning above, arriving on the first genuinely long value rather than at migration time. Note this probe asserts the byte limit, which belongs to the enforcing index, and not the duplicate-key number, which depends on whether Prisma built a constraint or an index; that distinction is probed separately below. 900 Unicode characters is 1800 bytes of index entry against the 1700 byte limit. Microsoft Learn lists 1946 at severity 16 with the text 'Operation failed. The index entry of length %d bytes for the index ... exceeds the maximum length of %d bytes for %S_MSG indexes' on the same errors page, checked 2026-09-03. The max cast on REPLICATE is deliberate: without it REPLICATE returns nvarchar(4000) and a longer probe would silently yield NULL and prove the opposite of its claim." + }, + { + "sql": "CREATE TABLE dbo.probe_prisma_sized (id nvarchar(64) NOT NULL PRIMARY KEY, email nvarchar(320) NOT NULL);\nGO\nCREATE UNIQUE NONCLUSTERED INDEX probe_prisma_sized_email_key ON dbo.probe_prisma_sized (email);\nGO\nINSERT INTO dbo.probe_prisma_sized (id, email) VALUES (N'a', REPLICATE(CAST(N'x' AS nvarchar(max)), 320));", + "expect": { "succeeds": true }, + "why": "The fix, asserted rather than recommended. This is what the skill's @db.NVarChar(64) and @db.NVarChar(320) model compiles to, confirmed by re-running migrate diff on it in the same 2026-09-03 session: NVARCHAR(64), NVARCHAR(320), NVARCHAR(max). The clustered key is 128 bytes against 900 and the nonclustered key is 640 against 1700, so the create emits no 1945 and the full-width insert that failed above succeeds. A succeeds-probe is right here because the claim genuinely is that nothing is raised." + }, + { + "sql": "CREATE TABLE dbo.probe_prisma_null (id int IDENTITY(1,1) NOT NULL, code nvarchar(64) NULL, CONSTRAINT probe_prisma_null_pkey PRIMARY KEY CLUSTERED (id), CONSTRAINT probe_prisma_null_code_key UNIQUE NONCLUSTERED (code));\nGO\nINSERT INTO dbo.probe_prisma_null (code) VALUES (NULL);\nINSERT INTO dbo.probe_prisma_null (code) VALUES (NULL);", + "expect": { "error": 2627 }, + "why": "The PostgreSQL habit that survives validation and fails at runtime: this engine treats NULL as a value for uniqueness, so a second null row in a String? @unique field is a duplicate key. The fixture is now the DDL Prisma actually emits, and that is the whole correction. An earlier version of this probe built the column with CREATE UNIQUE NONCLUSTERED INDEX and expected 2627; the engine returned 2601, because the two numbers name different objects. Learn gives 2601 as 'Cannot insert duplicate key row in object ... with unique index' (MSSQL_ENG002601) and 2627 as 'Violation of %ls constraint ... Cannot insert duplicate key in object' (MSSQL_ENG002627), checked 2026-09-03. Measured the same day on prisma 7.9.1: @unique and @@unique emit ADD CONSTRAINT ... UNIQUE NONCLUSTERED on BOTH the create-table and the alter-table path, never a standalone unique index, even though the generated migration labels the step -- CreateIndex. A UNIQUE constraint is enforced by an automatically created unique index (Learn, Unique constraints and check constraints), but the engine still reports the constraint, so 2627 is the number a reader hits when a Prisma @unique is violated and 2601 is the number they hit on an index they wrote by hand." + } + , + { + "sql": "CREATE TABLE dbo.probe_pr_uniq (id int IDENTITY, s nvarchar(20) NOT NULL);\nCREATE UNIQUE INDEX probe_pr_uniq_ix ON dbo.probe_pr_uniq (s);\nGO\nINSERT INTO dbo.probe_pr_uniq (s) VALUES (N'a');\nINSERT INTO dbo.probe_pr_uniq (s) VALUES (N'a');", + "expect": { + "error": 2601 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. What a Prisma @unique becomes and what a duplicate then costs. The number differs from the constraint form, which is why a log line can be read back to the shape Prisma actually emitted. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Use the prisma-azure-sql skill", + "Set up Prisma against Azure SQL Database", + "Write a Prisma schema for Azure SQL", + "Run Prisma migrations against my Azure SQL database" + ], + "implicit": [ + "The automatic creation of shadow databases is disabled on Azure SQL", + "P3020 when I run prisma migrate dev", + "prisma migrate dev works locally but fails against Azure", + "Field payload in model Thing can't be of type Json. The current connector does not support the Json type", + "my Prisma schema will not accept an enum", + "the datasource property url is no longer supported in schema files", + "inserting a long value fails on a column Prisma created for me" + ], + "routing": [ + "model my data with an ORM on Node and Azure SQL" + ], + "negative": [ + "connect my TypeScript API to Azure SQL with the mssql package", + "which Node driver should I use for Azure SQL", + "add EF Core to my .NET service on Azure SQL", + "wire SQLAlchemy up to Azure SQL from Python", + "retry logic for transient errors on Azure SQL", + "design the key and index strategy for a new schema", + "create the Entra administrator for my logical server" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql deleted file mode 120000 index 56fc4f8..0000000 --- a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/rag-on-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/SKILL.md new file mode 100644 index 0000000..b7cea9b --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/SKILL.md @@ -0,0 +1,311 @@ +--- +name: rag-on-azure-sql +description: >- + Builds retrieval augmented generation end to end on Azure SQL Database: chunking source text, + storing embeddings with the provenance that makes them re-runnable, retrieving with the filter + and the permission check inside the same query, and grounding an answer on what came back. Use + when someone asks to "build RAG on Azure SQL Database", "chat with my documents", "add semantic + search over my data", "keep embeddings in sync when rows change", "re-embed with a new model", + or "which chunks should I put in the prompt"; and when a retrieval pipeline returns plausible + but wrong context, or returns text the asking user is not allowed to read. This skill owns the + pipeline and the schema around it. The vector type, VECTOR_DISTANCE and the query shape are + vector-search-azure-sql, the embedding call embeddings-and-external-models, the offline + container path rag-local-with-container. +--- + +# Retrieval augmented generation on Azure SQL Database + +What tables exist, what is written when, what the retrieval query has to carry, and how to prove it +worked. + +Verified 2026-08-28 against a live Azure SQL Database, General Purpose serverless, compatibility +level 170, and re-run 2026-09-03, which is where `Msg 37579` comes from. Numbers not from those +runs are from Microsoft Learn, linked at the end. + +Open [references/pipeline.md](references/pipeline.md) when you sit down to write the ingest or the +re-embed: it holds the `MERGE`, the embedding loop, and what the two runs disagreed about. + +## What this owns + +The tables, the ingest, what the retrieval query carries, and the grounding. Open +`vector-search-azure-sql` before writing any similarity query, for the `vector` type's restrictions +and the shape that reaches the index; `embeddings-and-external-models` before steps 3 and 4, for +the external model and the chunking and embedding functions; `rls-multi-tenant` for predicates, +with step 6's caveat; `rag-local-with-container` when there is no cloud endpoint to develop on. + +## The correction + +An agent asked to build RAG here ports a vector store recipe: one table of chunk text with one +embedding column, retrieved by the application's identity, top k pasted into the prompt. It works +on the first demo. Two things are then wrong and **neither raises an error**. + +**1. The chunk table is a second copy of the source text.** Whatever protected the original does +not follow the copy. Row level security on `dbo.documents` does not apply to `dbo.document_chunks`, +and neither do its grants, its tenant column or its soft delete flag. A retrieval that is otherwise +perfect puts text the asking user may not read into a prompt, and the model quotes it back. The +obvious fix is refused: a security policy cannot reference a table carrying a vector index, +`Msg 37579`, so the chunk table cannot have a predicate at all and the retrieval query's `WHERE` +clause is the whole boundary. + +**2. Nothing records which model produced a stored vector.** A `vector(n)` column carries a +dimension and a base type and nothing else. The two most widely used embedding models both emit +1536 dimensions, so re-embedding half a corpus with another inserts cleanly and leaves two +incompatible geometries in one column. Distances across them are meaningless, no statement +fails, and the only symptom is that answers get worse. + +## 1. Fix the dimension before the schema + +The dimension follows the model, the ceiling is 1998, and it cannot be widened by `ALTER COLUMN` +even on an empty table. A 3072 dimension model does not fit and has to be asked for a shorter +vector at generation time. Write the number twice, as `vector(n)` and as the dimension +requested from the model; when they disagree the insert fails, the one loud failure here. + +## 2. The schema + +Two tables. Source rows stay authoritative and chunks are derived. + +```sql +CREATE TABLE dbo.documents ( + document_id INT NOT NULL IDENTITY PRIMARY KEY, + tenant_id INT NOT NULL, + title NVARCHAR(400) NOT NULL, + body NVARCHAR(MAX) NOT NULL, + updated_at DATETIME2(3) NOT NULL DEFAULT SYSUTCDATETIME() +); + +CREATE TABLE dbo.document_chunks ( + chunk_id INT NOT NULL IDENTITY PRIMARY KEY, -- clustered; the index needs one + document_id INT NOT NULL REFERENCES dbo.documents (document_id), + tenant_id INT NOT NULL, -- carried, not joined for, step 6 + ordinal INT NOT NULL, + chunk_text NVARCHAR(MAX) NOT NULL, + source_hash BINARY(32) NOT NULL, -- HASHBYTES over chunk_text + embed_model SYSNAME NOT NULL, -- provenance + embed_dims SMALLINT NOT NULL, + embedded_at DATETIME2(3) NULL, -- NULL exactly when embedding is + embedding VECTOR(1536) NULL +); + +CREATE UNIQUE INDEX ux_chunks_identity ON dbo.document_chunks (document_id, ordinal); +CREATE INDEX ix_chunks_tenant ON dbo.document_chunks (tenant_id); -- the retrieval filter +CREATE INDEX ix_chunks_pending ON dbo.document_chunks (chunk_id) + WHERE embedded_at IS NULL; -- the work queue +``` + +Four things are load bearing: + +- **`chunk_id` is a clustered primary key.** Without a clustered index the vector index cannot be + created at all, and that is discovered late. + +- **`source_hash` is how an ingest becomes idempotent.** There is no equality operator on a vector, + so a hash of the text is the only deduplication key. `HASHBYTES` takes the whole `nvarchar(max)` + chunk here, with no 8,000 byte limit. +- **`embed_model` and `embed_dims` are the provenance** failure 2 needs. They are the difference + between re-embedding what changed and re-embedding everything because nobody knows what is in + there. +- **`tenant_id` is carried on the chunk**, deliberately duplicated, so the retrieval filter needs + no join. It is not backed by a predicate here and cannot be. Step 6. + +`embedded_at` is NULL exactly when `embedding` is, because step 4 writes both in one statement, +which is what lets the work queue be a filtered index on an ordinary column. + +## 3. Chunk + +Chunk in the database when the text is already there, in the application when it arrives from +files. The chunking function and its size and overlap parameters are +`embeddings-and-external-models`. + +- **Chunk the same way every time.** Changing chunk size later invalidates every stored embedding + as surely as changing the model. +- **Keep the ordinal.** Grounding often needs a hit's neighbours, a cheap seek on + `(document_id, ordinal)` and impossible without it. +- **Store enough text to be an answer.** A chunk too small retrieves well and grounds badly. + +## 4. Embed in restartable batches + +The model call is `embeddings-and-external-models`. The pipeline shape: + +```sql +UPDATE TOP (200) c + SET embedding = /* embeddings-and-external-models: the model call over c.chunk_text */, + embed_model = @embed_model, + embed_dims = @embed_dims, + embedded_at = SYSUTCDATETIME() + FROM dbo.document_chunks AS c + WHERE c.embedded_at IS NULL; +``` + +Loop until it affects zero rows. The filtered index makes that predicate a seek over a corpus that +is mostly done, where a set based update over the whole table re-embeds finished work and costs +money. Write the vector and its three provenance columns in the **same** statement: a vector +without provenance is what failure 2 is made of. It restarts safely because the identity is +`(document_id, ordinal)`. + +## 5. The index goes in after the load + +```sql +CREATE VECTOR INDEX vi_document_chunks + ON dbo.document_chunks (embedding) + WITH (METRIC = 'cosine', TYPE = 'diskann'); +``` + +- **At least 100 rows carrying a non-null vector**, `Msg 42266` below that. So it belongs to the + load, never the schema deployment: a package import creates objects before it loads rows. Drop + the index before an export, create it after. +- **The metric is fixed at build time** and the retrieval query has to name the same one. +- **Everything works before the index exists**, exactly and slowly, which is why an environment + runs without one and only latency says so. +- **`TRUNCATE TABLE` is refused once it exists**, `Msg 42232`. Drop, truncate, reload, recreate. + +## 6. Retrieve with the filter in the query + +One query does the similarity, the tenant filter and the ordering the prompt needs. + +```sql +DECLARE @q VECTOR(1536) = /* the question, embedded with the SAME model as the corpus */; + +SELECT chunk_id, document_id, ordinal, chunk_text, distance -- reading order, for the prompt +FROM ( + SELECT TOP (8) WITH APPROXIMATE + c.chunk_id, c.document_id, c.ordinal, c.chunk_text, r.distance + FROM VECTOR_SEARCH( + TABLE = dbo.document_chunks AS c, + COLUMN = embedding, + SIMILAR_TO = @q, + METRIC = 'cosine' + ) AS r + WHERE c.tenant_id = @tenant_id -- applied DURING the search, see below + ORDER BY r.distance +) AS hits +ORDER BY document_id, ordinal; +``` + +- **Embed the question with the model that embedded the corpus.** A question embedded by a + different model returns results ranked by nothing. +- **The filter belongs inside, not after.** On a current index version the predicate is applied + during the search, so a filtered top eight returns eight whenever eight qualify. On an earlier + index version the search runs before any predicate: the eight nearest are chosen first and the + filter then removes some, returning three, or none, with qualifying rows unread. Check 3 says + which one you have. +- **The inner `ORDER BY` takes the distance column ascending and nothing else.** Adding + `c.ordinal` fails with `Msg 42271` and dropping it fails with `Msg 42248`, which is why reading + order is applied in the outer query. +- **No policy can back that `WHERE` clause up.** `CREATE SECURITY POLICY` on a table carrying a + vector index is refused, `Msg 37579`, measured 2026-09-03 and documented nowhere on Microsoft + Learn. Keep the policy on `dbo.documents`, where it still guards direct reads, and treat the + filter above as security code: test that removing it changes the result. If a predicate on the + retrieval path is non-negotiable, the sourced ways out are to drop the vector index, since + `VECTOR_SEARCH` still runs as a brute force scan without one, or to split the table so the index + sits on an embeddings table carrying no policy and the protected text joins back, costing the + filter its place inside the search. +- **Retrieve more than you ground on** and re-rank the candidates, but never substitute a bigger k + for a missing filter. + +## 7. Ground the answer + +- **Pass the chunk text, the document identifier and the ordinal** so the answer can cite it and a + human can check the citation. +- **Give the model permission to say the context does not contain the answer.** A pipeline that + always answers is a hallucination pipeline with extra steps. +- **Do not paste the distance in as a confidence.** It is a geometric distance, not a probability, + and its scale differs by metric. +- **Log the retrieved chunk identifiers with the answer.** When a wrong answer is reported, the + only useful question is what was retrieved, and nothing else answers it later. + +## 8. Keep it current + +**When a source row changes**, delete its chunks and re-chunk it: boundaries move when text +changes, so updating in place leaves orphans. `MERGE` on `(document_id, ordinal)` does it, and is +supported on a table carrying a current vector index, which maintains itself. The statement is in +the reference. + +**When the model or the chunking changes**, everything is stale at once. Add a new column with its +own provenance, backfill fully while retrieval still runs on the old, cut over, then drop the old +column and index. A half migrated column is failure 2. Rebuild the vector index afterwards: the +graph was built for the old distribution and recall degrades with no error. + +## Check it worked + +Four checks. The second decides whether the retrieval is safe to expose. + +```sql +-- 1. Provenance: one model across the corpus, and nothing left unembedded. +SELECT embed_model, embed_dims, COUNT(*) AS chunks, + SUM(CASE WHEN embedding IS NULL THEN 1 ELSE 0 END) AS unembedded +FROM dbo.document_chunks +GROUP BY embed_model, embed_dims; + +-- 2. Which tables have a predicate and which have a vector index. Never both. +SELECT t.name AS table_name, + MAX(CASE WHEN sp.target_object_id IS NULL THEN 0 ELSE 1 END) AS has_policy, + MAX(CASE WHEN vi.object_id IS NULL THEN 0 ELSE 1 END) AS has_vector_index +FROM sys.tables AS t +LEFT JOIN sys.security_predicates AS sp ON sp.target_object_id = t.object_id +LEFT JOIN sys.vector_indexes AS vi ON vi.object_id = t.object_id +GROUP BY t.name; + +-- 3. The vector index exists and is the current version. +SELECT OBJECT_NAME(v.object_id) AS table_name, i.name AS index_name, + JSON_VALUE(v.build_parameters, '$.Version') AS index_version +FROM sys.vector_indexes AS v +JOIN sys.indexes AS i ON v.object_id = i.object_id AND v.index_id = i.index_id; + +-- 4. Every stored hash still agrees with the text it was taken from. +SELECT COUNT(*) AS hashes_disagreeing_with_their_text +FROM dbo.document_chunks +WHERE source_hash <> HASHBYTES('SHA2_256', chunk_text); +``` + +Run them non-interactively and keep the output: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 \ + -i check-rag-pipeline.sql -o check-rag-pipeline.out +``` + +Expected: check 1 returns **exactly one row** with `unembedded` `0`; two rows means two geometries +in one column and every distance across them meaningless, and a non-zero `unembedded` means part of +the corpus is invisible. Check 2 shows `documents` with `has_policy` `1`, and `document_chunks` with +`has_vector_index` `1` and `has_policy` `0`; that zero is the engine's rule, not an omission to fix, +which is why step 6's `WHERE` clause is what needs the test. Check 3 returns one row per vector +index with `index_version` `3`; empty means every retrieval is scanning, and an older version means +step 6's filter runs after the search rather than during it. Check 4 returns `0`; anything else +means the deduplication key is wrong and re-runs are re-embedding unchanged text. + +`-m-1` makes a severity 10 message print its number, and `-b` only sets a non-zero exit at severity +11 and above, so `sqlcmd` exiting 0 says nothing about the four. Read the output file. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build a severity 10 message arrives with no number to read. `build-app-on-azure-sql` tells the two +builds apart in one table. + +## Do not + +- Do not assume the copy inherits the source's protection, and do not try to give it a policy of + its own: that is refused. The query's filter is the boundary and needs a test that fails without + it, in the database rather than in application code. +- Do not store an embedding without recording which model made it, and do not mix two models' + vectors in one column. Nothing will tell you. +- Do not embed the question with a different model from the corpus. +- Do not re-embed rows that have not changed. Hash the text and skip them. +- Do not treat a distance as a confidence score, and do not put it in the prompt. +- Do not create the vector index as part of a schema-only deployment, and do not ship a pipeline + that cannot say "the retrieved context does not answer this". + +## References + +- [references/pipeline.md](references/pipeline.md): open it while writing step 2 onwards, or when a + claim above disagrees with what you are seeing. It has the idempotent `MERGE`, the embedding + loop, the mixed corpus checks, and the `Msg 37579` disagreement in full. +- [VECTOR_SEARCH](https://learn.microsoft.com/sql/t-sql/functions/vector-search-transact-sql): read + when a filtered retrieval returns fewer rows than asked for, for iterative versus post filtering, + and for the `ORDER BY` rules behind `Msg 42248` and `Msg 42271`. +- [CREATE VECTOR INDEX limitations](https://learn.microsoft.com/sql/t-sql/statements/create-vector-index-transact-sql#limitations-and-considerations): + read before promising a policy on an indexed table. As of 2026-09-03 it lists partitioning, + `TRUNCATE` and package import, and not `Msg 37579`. +- [Row level security](https://learn.microsoft.com/sql/relational-databases/security/row-level-security): + read before putting a predicate on `dbo.documents`. +- [HASHBYTES](https://learn.microsoft.com/sql/t-sql/functions/hashbytes-transact-sql): read before + choosing a deduplication key, for the algorithms still supported and the input limit. diff --git a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/references/pipeline.md b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/references/pipeline.md new file mode 100644 index 0000000..846ab5c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/references/pipeline.md @@ -0,0 +1,175 @@ +# The pipeline, in full + +## Contents + +- [What was measured](#what-was-measured) +- [Schema](#schema) +- [Idempotent ingest](#idempotent-ingest) +- [Embedding pass](#embedding-pass) +- [Vector index, after the load](#vector-index-after-the-load) +- [Retrieval](#retrieval) +- [Row level security and the vector index](#row-level-security-and-the-vector-index) +- [Detecting a mixed corpus](#detecting-a-mixed-corpus) +- [Changing the model or the chunking](#changing-the-model-or-the-chunking) +- [Failure modes and what they look like](#failure-modes-and-what-they-look-like) + +## What was measured + +Run on 2026-08-28 against an Azure SQL Database provisioned for it: General Purpose serverless, +two vCores, compatibility level 170, Microsoft Entra authentication only, and re-run 2026-09-03 as +this skill's probes, which is where `Msg 37579` comes from. Statements below ran as written, apart +from the embedding call, which is `embeddings-and-external-models` territory and is a placeholder. + +Three results decide the design: + +1. A security policy **cannot be created at all** on a table carrying a vector index, `Msg 37579`, + measured 2026-09-03. This reverses what this file claimed on 2026-08-28 and changes the advice + in step 6. See below. +2. A vector column's dimension could not be changed by `ALTER COLUMN` even on an empty table, and + `CREATE VECTOR INDEX` failed on 50 rows and succeeded on 500. Both are `vector-search-azure-sql` + and are probed there; they are noted here only because they decide the schema. +3. Nothing about the queries changed when the index appeared, which is why an environment can + silently run without one and only latency says so. + +## Schema + +The canonical schema is `SKILL.md` step 2, deliberately not repeated here: two copies of a +`CREATE TABLE` drift, and this pair already had, with `embedded_at` declared `NOT NULL` in one file +and `NULL` in the other while the ingest below never supplied it. + +`source_hash` is `HASHBYTES('SHA2_256', chunk_text)`. The rest of the reasoning is in that step. + +## Idempotent ingest + +For a freshly chunked set for one document in `@incoming (ordinal, chunk_text)`: + +```sql +MERGE dbo.document_chunks AS target +USING ( + SELECT @document_id AS document_id, + @tenant_id AS tenant_id, + i.ordinal, + i.chunk_text, + HASHBYTES('SHA2_256', i.chunk_text) AS source_hash + FROM @incoming AS i +) AS source + ON target.document_id = source.document_id + AND target.ordinal = source.ordinal +WHEN MATCHED AND target.source_hash <> source.source_hash THEN + UPDATE SET chunk_text = source.chunk_text, + source_hash = source.source_hash, + embedding = NULL, -- text changed, so the vector is stale + embedded_at = NULL +WHEN NOT MATCHED BY TARGET THEN + INSERT (document_id, tenant_id, ordinal, chunk_text, source_hash, embed_model, embed_dims) + VALUES (source.document_id, source.tenant_id, source.ordinal, source.chunk_text, + source.source_hash, @embed_model, @embed_dims) +WHEN NOT MATCHED BY SOURCE AND target.document_id = @document_id THEN + DELETE; -- the document got shorter +``` + +The `WHEN MATCHED` branch is the point: unchanged text is left alone, so a re-run costs nothing. `MERGE` against a table carrying a current vector index is supported, and Learn +lists full `INSERT`, `UPDATE`, `DELETE` and `MERGE` support with real time index maintenance as a +property of the latest index version specifically. + +## Embedding pass + +The batched `UPDATE` is `SKILL.md` step 4. Loop until it affects zero rows. The filtered index makes the predicate a seek rather than a scan +over a mostly embedded corpus. + +## Vector index, after the load + +`CREATE VECTOR INDEX`, the statement in `SKILL.md` step 5, fails on the 100 row minimum until the embedding pass has produced 100 non-null vectors. That +minimum is why the index cannot be part of a data-tier package import, which creates objects before +it loads rows. + +## Retrieval + +The query is `SKILL.md` step 6. The predicate sits inside it on purpose. +[VECTOR_SEARCH](https://learn.microsoft.com/sql/t-sql/functions/vector-search-transact-sql) states +that a current index version applies `WHERE` predicates **during** the search and an earlier one +applies them **after**, so on an earlier index the top eight is chosen first and the filter then +removes some of it. Read `index_version` before trusting a filtered top k to be full. The same page +fixes the `ORDER BY`: the distance column ascending and nothing else, `Msg 42271` otherwise, and +`Msg 42248` with no `ORDER BY`. Sort into reading order in an outer query. + +To fetch the neighbours of a hit for a wider grounding window: + +```sql +SELECT ordinal, chunk_text +FROM dbo.document_chunks +WHERE document_id = @document_id + AND ordinal BETWEEN @ordinal - 1 AND @ordinal + 1 +ORDER BY ordinal; +``` + +## Row level security and the vector index + +These two statements are **refused** while `dbo.document_chunks` carries a vector index: + +```sql +CREATE FUNCTION dbo.fn_chunk_tenant (@tenant_id INT) +RETURNS TABLE +WITH SCHEMABINDING +AS + RETURN SELECT 1 AS is_visible + WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS INT); +GO + +CREATE SECURITY POLICY dbo.chunk_access + ADD FILTER PREDICATE dbo.fn_chunk_tenant (tenant_id) ON dbo.document_chunks + WITH (STATE = ON); +``` + +```output +Msg 37579, Level 16, State 1 +The security policy 'dbo.chunk_access' cannot reference tables with vector indexes. +Table 'dbo.document_chunks' has a vector index. +``` + +The same two statements are correct against `dbo.documents`, which carries no vector index, and +that policy does **not** reach the chunk copy. So on the retrieval path the tenant boundary is the +query's `WHERE` clause and nothing else. Three sourced ways to live with that: + +1. Keep the filter in the query and wrap a test around it: run the retrieval with the filter and + without it, and require different row counts. That test is security code now. +2. Drop the vector index on the tenant-scoped table. `VECTOR_SEARCH` still runs without one as a + brute force scan and a policy can then be created, which gives back what the index was for. +3. Split the table: the index on an embeddings table with no policy, the chunk text and tenant + column on a policy-bearing table joined back, the shape of Learn's multi-table example. The + predicate then sits on the joined table, and Learn labels only the searched table's predicate an + iterative filter, so expect post-filter behaviour. + +**This limitation is not documented.** Learn's `CREATE VECTOR INDEX` limitations list names +partitioning, the clustered primary key, replication, `TRUNCATE TABLE` and package import, and the +`vector` type's limitations page names constraints, indexes, ledger tables and Always Encrypted. +Neither mentions security policies, as of 2026-09-03. `Msg 37579` is this skill's own measurement, +and it contradicts what an earlier revision of this file recorded on 2026-08-28. That entry could +not be re-run, the refusal reproduces, so the refusal stands. Creating the policy **before** the +index was not measured, so do not plan around that order. + +## Detecting a mixed corpus + +Check 1 in `SKILL.md` is the query. One row is healthy. Two or more means distances are compared across incompatible geometries and no +statement anywhere will have failed. Put this in a scheduled check, not a runbook. + +## Changing the model or the chunking + +Add a second column, `embedding_v2 VECTOR(1024)` say, with its own provenance columns, because the +existing one cannot be altered to a different dimension even when empty. Backfill it completely +with retrieval still running against the old column, create its vector index, which coexists +because there is one per column, cut the query over, then drop the old index and columns. + +For a near total in-place replacement, drop and recreate the vector index after the load: the graph +was built for the previous distribution and recall degrades with no error. + +## Failure modes and what they look like + +| Symptom | Cause | Where it is caught | +|---|---|---| +| Answers quote documents the user should not see | The retrieval query has no tenant filter, and no policy can supply one | Nowhere, until someone reports it. Only a test on the query catches it | +| Answers get vaguer after a model change | Two models' vectors in one column | The group by above | +| Retrieval is correct but slow, and slows as the corpus grows | The query shape never reaches the vector index | The execution plan. `vector-search-azure-sql` | +| Filtered results return fewer rows than asked for | The filter runs in application code, or the index is an earlier version that post-filters | `index_version`, and the count with the predicate moved into the query | +| Part of the corpus is never retrieved | Rows with a null embedding | The unembedded count above | +| `CREATE SECURITY POLICY` is refused | The target table carries a vector index | `Msg 37579`, at once. There is no policy on the retrieval path; the query's filter is the boundary | diff --git a/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..b06eb11 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/rag-on-azure-sql/skill.spec.jsonc @@ -0,0 +1,211 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "rag-on-azure-sql", + "domain": "ai-vector-and-rag", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Agents build retrieval augmented generation on Azure SQL Database by porting a vector store recipe: one chunk table holding extracted text and one embedding column, retrieved with the application's identity, top k pasted into the prompt. It demonstrates correctly and then fails in two ways that raise nothing. First, the chunk table is a second copy of the source text, so the row level security policy, the tenant column and the grants that protected the original table do not apply to it, and a retrieval that is otherwise perfect puts text the asking user may not read into the prompt where the model quotes it back; and the obvious repair is not available: measured 2026-09-03, CREATE SECURITY POLICY against a table carrying a vector index is refused outright with Msg 37579, 'the security policy cannot reference tables with vector indexes', so the chunk table cannot be given a predicate at all and the tenant filter inside the retrieval query is the entire boundary, held up by a test rather than by the engine. Microsoft Learn documents neither the refusal nor the error number, on the CREATE VECTOR INDEX limitations list or on the vector type's limitations page, as of 2026-09-03. Second, nothing records which model produced each stored vector, because a vector column carries a dimension count and a base type and nothing else, and the two most widely used embedding models both emit 1536 dimensions, so re-embedding part of a corpus with a different model inserts cleanly, matches on dimension and leaves two incompatible geometries in one column whose only symptom is that answers get worse. Being wrong costs a data leak that reads as a correct answer and a corpus whose retrieval quality decays with nothing to point at.", + "value_probe": { + "gate": "The answer treats an engine enforced predicate on the indexed chunk table as unavailable and puts the tenant boundary somewhere that actually holds, rather than handing over a policy definition the engine will refuse, and it explains the quality drop as two incompatible sets of vectors in one column.", + "task": "We run a multi-tenant support desk product on Azure SQL Database. Every customer's documents live in dbo.documents, which is protected so a tenant can only ever read its own rows. For the new assistant feature we built dbo.doc_chunks, holding chunk text, tenant_id and an embedding column, with the index that makes retrieval fast, and the app pulls the closest chunks and pastes them into the prompt.\n\nTwo problems. First, our security reviewer will not sign off until the tenant boundary on the chunk table is enforced by the database itself rather than by a WHERE clause the application has to remember. Second, about a third of the corpus was re-embedded a few weeks ago after we switched to a newer model with the same output size, and since then the assistant cites the wrong document noticeably more often, though nothing errors and every row count is right.\n\nTell us how to enforce the boundary, and what is going on with the answer quality.", + "states": [ + "The engine refuses to attach a security policy to a table that carries a vector index, so that repair is not available on this table.", + "With the engine side predicate unavailable, the tenant filter written into the retrieval query is the entire boundary and is held up by review and tests rather than by the engine.", + "The chunk table is a second copy of the source text and inherits none of the protections that guarded the original table.", + "A vector column records only a dimension count and a base type, so nothing in the schema records which model produced each stored vector.", + "Vectors from two different embedding models of the same width insert cleanly and coexist in one column, and the only symptom is that answers get worse." + ], + "refutes": [ + "A security policy with a filter predicate can be created on this chunk table to enforce the tenant boundary despite the vector index on it.", + "Two embeddings with the same number of dimensions are interchangeable, so mixing the output of two models in one column is harmless.", + "The chunk table inherits the predicates, grants and protections of the source table the text was extracted from.", + "A retrieval that returns text the asking user is not entitled to read will surface as an error or an empty result rather than as a confident answer." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db"], + "related": [ + "vector-search-azure-sql", + "embeddings-and-external-models", + "rag-local-with-container", + "langchain-and-llamaindex-on-azure-sql", + "chunking-strategies", + "design-azure-sql-schema", + "connect-to-azure-sql", + "rls-multi-tenant", + "full-text-and-hybrid-ranking", + "diagnose-vector-recall-drop", + "azuresql-db-rag" + ], + "validation": { + "target": "both", + "setup": [ + "a database reachable with permission to create tables, indexes and a security policy", + "a source table of documents and an embedding model whose output dimension the engine can store" + ], + "assert": [ + "the declared vector dimension equals the dimension requested from the model, and is within the engine's 1998 ceiling", + "every embedded row carries the model name, the dimension and a timestamp, and grouping by the model column returns exactly one row", + "the chunk table has a clustered primary key, a unique key on the chunk identity, and an index on the columns retrieval filters by", + "re-running the ingest over unchanged source text embeds nothing", + "the retrieval query carries the permission filter itself, and a test fails when that filter is removed, because no security policy can be created on the vector-indexed table to back it up", + "the question is embedded with the same model as the corpus", + "the vector index is created after the load rather than imported as part of a schema package", + "the retrieved chunk identifiers are logged alongside the generated answer", + "no source document or test fixture in the repository contains real customer data" + ], + "probes": + [ + { + "sql": "CREATE TABLE dbo.probe_rag_docs (document_id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, tenant_id int NOT NULL, body nvarchar(max) NOT NULL);\nCREATE TABLE dbo.probe_rag_chunks (\n chunk_id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED,\n document_id int NOT NULL REFERENCES dbo.probe_rag_docs (document_id),\n tenant_id int NOT NULL,\n ordinal int NOT NULL,\n chunk_text nvarchar(max) NOT NULL,\n source_hash binary(32) NOT NULL,\n embed_model sysname NOT NULL,\n embed_dims smallint NOT NULL,\n embedded_at datetime2(3) NULL,\n embedding vector(1536) NULL);\nCREATE UNIQUE INDEX ux_probe_rag_identity ON dbo.probe_rag_chunks (document_id, ordinal);\nCREATE INDEX ix_probe_rag_tenant ON dbo.probe_rag_chunks (tenant_id);\nCREATE INDEX ix_probe_rag_pending ON dbo.probe_rag_chunks (chunk_id) WHERE embedded_at IS NULL;\nSELECT COUNT(*) AS work_queue_index_is_filtered\nFROM sys.indexes\nWHERE name = 'ix_probe_rag_pending' AND has_filter = 1;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The whole of step 2, made executable: the two table schema with the vector column, the identity key and the three indexes is creatable exactly as printed. It also pins the one design choice a reader is likely to undo, which is that the work queue is a filtered index on embedded_at and not on the vector column, so its predicate is an ordinary column and no claim about filtering on a vector is being made. An earlier revision of this skill declared embedded_at NOT NULL in SKILL.md and NULL in its reference file while the ingest supplied neither, which this probe would have caught." + }, + { + "sql": "DECLARE @chunk nvarchar(max) = REPLICATE(CAST(N'x' AS nvarchar(max)), 20000);\nSELECT DATALENGTH(@chunk) AS input_bytes,\n DATALENGTH(HASHBYTES('SHA2_256', @chunk)) AS hash_bytes;", + "expect": { + "rows": [ + [ + 40000, + 32 + ] + ] + }, + "why": "source_hash is the only deduplication key available, because there is no equality operator on a vector, so it has to survive a real chunk rather than a short one. Microsoft Learn's HASHBYTES page scopes the 8,000 byte input limit to releases older than this engine and gives SHA2_256 as 256 bits, which is the 32 here. The CAST on the first argument is deliberate: REPLICATE(N'x', 20000) without it returns nvarchar(4000) and the probe would silently measure a short string." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_src (id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL);\nCREATE TABLE dbo.probe_rag_copy (id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL);\nINSERT INTO dbo.probe_rag_src (id, tenant_id) VALUES (1,1),(2,1),(3,2),(4,2);\nINSERT INTO dbo.probe_rag_copy (id, tenant_id) VALUES (1,1),(2,1),(3,2),(4,2);\nGO\nCREATE FUNCTION dbo.fn_probe_rag_src_tenant (@tenant_id int)\nRETURNS TABLE WITH SCHEMABINDING\nAS RETURN SELECT 1 AS is_visible WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int);\nGO\nCREATE SECURITY POLICY dbo.probe_rag_src_policy\n ADD FILTER PREDICATE dbo.fn_probe_rag_src_tenant (tenant_id) ON dbo.probe_rag_src\n WITH (STATE = ON);\nGO\nEXEC sys.sp_set_session_context @key = N'tenant_id', @value = 1;\nSELECT (SELECT COUNT(*) FROM dbo.probe_rag_src) AS visible_in_the_source,\n (SELECT COUNT(*) FROM dbo.probe_rag_copy) AS visible_in_the_copy;", + "expect": { + "rows": [ + [ + 2, + 4 + ] + ] + }, + "why": "Failure 1 with nothing else in the way. The two tables hold identical rows for two tenants and the policy is on one of them, which is what a pipeline that chunks dbo.documents into dbo.document_chunks and protects only the first has built. Two rows are visible in the protected table and all four in the copy, with no error and no warning anywhere. Microsoft Learn's row level security page states that a policy applies to all users including dbo, so this leak is not an artifact of the probe running with an administrative connection." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_rls (chunk_id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, tenant_id int NOT NULL, embedding vector(5) NOT NULL);\nINSERT INTO dbo.probe_rag_rls (tenant_id, embedding)\nSELECT (value % 2) + 1,\n CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 200);\nGO\nCREATE VECTOR INDEX vi_probe_rag_rls ON dbo.probe_rag_rls (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nCREATE FUNCTION dbo.fn_probe_rag_tenant (@tenant_id int)\nRETURNS TABLE WITH SCHEMABINDING\nAS RETURN SELECT 1 AS is_visible WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int);\nGO\nCREATE SECURITY POLICY dbo.probe_rag_policy\n ADD FILTER PREDICATE dbo.fn_probe_rag_tenant (tenant_id) ON dbo.probe_rag_rls\n WITH (STATE = ON);", + "expect": { + "error": 37579 + }, + "why": "The repair a reader reaches for, refused by the engine. An earlier revision of this skill asserted the opposite, that a filter predicate on the chunk table is honoured through approximate vector search, and this probe run on 2026-09-03 answered Msg 37579, 'the security policy cannot reference tables with vector indexes'. That measurement is why step 6 now says the query's WHERE clause is the whole tenant boundary and has to carry a test. Microsoft Learn documents neither the refusal nor the number: as of 2026-09-03 the CREATE VECTOR INDEX limitations list names partitioning, the clustered primary key, replication, TRUNCATE TABLE and package import, and the vector type's limitations page names constraints, indexes, ledger tables and Always Encrypted, and neither mentions security policies. So if this probe ever passes, the product moved and the skill can be relaxed rather than the probe being wrong. Creating the policy BEFORE the index was not measured and nothing here claims anything about that order." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_merge (\n document_id int NOT NULL, ordinal int NOT NULL,\n chunk_text nvarchar(max) NOT NULL, source_hash binary(32) NOT NULL,\n embedded_at datetime2(3) NULL,\n PRIMARY KEY CLUSTERED (document_id, ordinal));\nDECLARE @incoming TABLE (ordinal int, chunk_text nvarchar(max));\nINSERT INTO @incoming (ordinal, chunk_text) VALUES (1, N'alpha'), (2, N'beta');\nMERGE dbo.probe_rag_merge AS target\nUSING (SELECT 1 AS document_id, i.ordinal, i.chunk_text, HASHBYTES('SHA2_256', i.chunk_text) AS source_hash FROM @incoming AS i) AS source\n ON target.document_id = source.document_id AND target.ordinal = source.ordinal\nWHEN MATCHED AND target.source_hash <> source.source_hash THEN\n UPDATE SET chunk_text = source.chunk_text, source_hash = source.source_hash, embedded_at = NULL\nWHEN NOT MATCHED BY TARGET THEN\n INSERT (document_id, ordinal, chunk_text, source_hash) VALUES (source.document_id, source.ordinal, source.chunk_text, source.source_hash);\nUPDATE dbo.probe_rag_merge SET embedded_at = SYSUTCDATETIME();\nMERGE dbo.probe_rag_merge AS target\nUSING (SELECT 1 AS document_id, i.ordinal, i.chunk_text, HASHBYTES('SHA2_256', i.chunk_text) AS source_hash FROM @incoming AS i) AS source\n ON target.document_id = source.document_id AND target.ordinal = source.ordinal\nWHEN MATCHED AND target.source_hash <> source.source_hash THEN\n UPDATE SET chunk_text = source.chunk_text, source_hash = source.source_hash, embedded_at = NULL\nWHEN NOT MATCHED BY TARGET THEN\n INSERT (document_id, ordinal, chunk_text, source_hash) VALUES (source.document_id, source.ordinal, source.chunk_text, source.source_hash);\nSELECT COUNT(*) AS chunks,\n SUM(CASE WHEN embedded_at IS NULL THEN 1 ELSE 0 END) AS re_queued_for_embedding\nFROM dbo.probe_rag_merge;", + "expect": { + "rows": [ + [ + 2, + 0 + ] + ] + }, + "why": "Idempotency, which is the difference between re-embedding what changed and re-embedding a corpus every night for money. The ingest runs twice over identical text with the embedding pass simulated in between, and nothing is queued again, because the WHEN MATCHED branch is guarded by the hash rather than firing on every match. A skill that says an ingest should be idempotent and never shows the guard has left the reader to discover that an unguarded UPDATE nulls every vector it touches." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_order (chunk_id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, ordinal int NOT NULL, embedding vector(5) NOT NULL);\nINSERT INTO dbo.probe_rag_order (ordinal, embedding)\nSELECT value,\n CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX vi_probe_rag_order ON dbo.probe_rag_order (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]';\nSELECT TOP (5) WITH APPROXIMATE t.chunk_id, t.ordinal, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_rag_order AS t, COLUMN = embedding, SIMILAR_TO = @q, METRIC = 'cosine') AS r\nORDER BY r.distance, t.ordinal;", + "expect": { + "error": 42271 + }, + "why": "The instinct this pipeline produces and the engine refuses. Grounding wants the retrieved chunks in reading order, so an author adds the ordinal to the ORDER BY of the retrieval query, and Microsoft Learn's VECTOR_SEARCH page states that TOP WITH APPROXIMATE requires ORDER BY on the distance column ascending and no other columns, Msg 42271, with Msg 42248 when ORDER BY is dropped altogether. That is why step 6 sorts into reading order in an outer query instead." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_mixed (chunk_id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, embed_model sysname NOT NULL, embed_dims smallint NOT NULL, embedding vector(3) NOT NULL);\nINSERT INTO dbo.probe_rag_mixed (embed_model, embed_dims, embedding)\nVALUES (N'model-a', 3, '[1,2,3]'), (N'model-a', 3, '[1,2,4]'), (N'model-b', 3, '[4,5,6]');\nSELECT COUNT(*) AS distinct_model_dimension_pairs\nFROM (SELECT embed_model, embed_dims FROM dbo.probe_rag_mixed GROUP BY embed_model, embed_dims) AS m;", + "expect": { + "rows": [ + [ + 2 + ] + ] + }, + "why": "Failure 2, and the reason check 1 in the skill exists. A second model's vectors go into the same column with no error, no warning and no dimension complaint, because the column carries a dimension and a base type and nothing else. The provenance columns are the only thing in the database that can see it, and this probe is the group by from the skill returning the two rows that mean every distance across them is meaningless." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_pol (id int NOT NULL PRIMARY KEY, tenant_id int NOT NULL);\nGO\nCREATE FUNCTION dbo.fn_probe_rag_pol (@tenant_id int)\nRETURNS TABLE WITH SCHEMABINDING\nAS RETURN SELECT 1 AS is_visible WHERE @tenant_id = CAST(SESSION_CONTEXT(N'tenant_id') AS int);\nGO\nCREATE SECURITY POLICY dbo.probe_rag_pol_policy\n ADD FILTER PREDICATE dbo.fn_probe_rag_pol (tenant_id) ON dbo.probe_rag_pol\n WITH (STATE = ON);\nGO\nSELECT OBJECT_NAME(sp.target_object_id) AS protected_table, sp.predicate_type_desc, p.is_enabled\nFROM sys.security_predicates AS sp\nJOIN sys.security_policies AS p ON p.object_id = sp.object_id\nWHERE OBJECT_NAME(sp.target_object_id) = 'probe_rag_pol';", + "expect": { + "rows": [ + [ + "probe_rag_pol", + "FILTER", + 1 + ] + ] + }, + "why": "Check 2 in the skill is the one that catches failure 1 before a user does, so the diagnostic itself is asserted rather than only the defect it finds. The column names and the join come from Microsoft Learn's sys.security_predicates and sys.security_policies pages: target_object_id is the protected table, predicate_type_desc is FILTER or BLOCK, and is_enabled lives on the policy rather than the predicate, which is the mistake that would make the check report a disabled policy as enforcing." + } + , + { + "sql": "CREATE TABLE dbo.probe_rag_99 (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_rag_99 (v) SELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 99);\nGO\nCREATE VECTOR INDEX probe_rag_99_ix ON dbo.probe_rag_99 (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42266 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The 100 row minimum, which is why the index belongs to the load step and not to the schema. A fresh database has no index until it has content and every query is silently exact until then. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_tr (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_rag_tr (v) SELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_rag_tr_ix ON dbo.probe_rag_tr (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nTRUNCATE TABLE dbo.probe_rag_tr;", + "expect": { + "error": 42232 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. TRUNCATE is refused once the index exists, which is the step a re-embedding pipeline written before the index was added will hit. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_sk (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, ordinal int NOT NULL, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_rag_sk (ordinal, v) SELECT value, CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_rag_sk_ix ON dbo.probe_rag_sk (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(3) = N'[1,1,2]';\nSELECT TOP (3) WITH APPROXIMATE t.id, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_rag_sk AS t, COLUMN = v, SIMILAR_TO = @q, METRIC = 'cosine') AS r\nORDER BY r.distance, t.ordinal;", + "expect": { + "error": 42271 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Adding the chunk ordinal as a second sort key is refused. The skill's point is that reading chunks back in order has to happen outside the approximate search, and this is the number that says so. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_rag_ap (id int NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nSELECT TOP (2) WITH APPROXIMATE id FROM dbo.probe_rag_ap ORDER BY id;", + "expect": { + "error": 42248 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. And dropping the VECTOR_SEARCH instead is refused too, so neither half of the obvious repair works. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Use the rag-on-azure-sql skill", + "Build a retrieval augmented generation pipeline on Azure SQL Database", + "Let users chat with our document library stored in Azure SQL Database", + "Add semantic search over the documents table in Azure SQL Database", + "How do I keep embeddings in sync when the source rows change" + ], + "implicit": [ + "we switched embedding models last month and the answers got worse for no reason", + "our assistant quoted a document that user was not supposed to see", + "how many chunks should I put in the prompt and how do I cite them", + "do I re-embed everything every night or only what changed", + "where should the tenant filter go in a retrieval query", + "we want to replace our separate vector database and keep the documents in Azure SQL Database", + "the assistant answers confidently even when the retrieved context says nothing useful", + "how do I re-embed a whole corpus with a new model without downtime" + ], + "routing": [ + "add a question answering feature over our own content on Azure SQL Database", + "design the tables for a document assistant backed by Azure SQL Database", + "productionise a RAG prototype that currently keeps embeddings in files" + ], + "negative": [ + "What are the restrictions on the vector data type in Azure SQL Database", + "Why does my similarity query scan the whole table instead of using the vector index", + "Show me the syntax for VECTOR_DISTANCE and the distance metrics", + "Create an external model and call the embedding function from T-SQL", + "Which permissions does an external model endpoint need and how does the allowlist work", + "Run a local RAG prototype offline against the Azure SQL Database container", + "Build a RAG scenario on the container", + "Set up full text search and rank the results", + "Add retry and connection pooling to my data access layer", + "Design the schema for my orders and customers tables" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql deleted file mode 120000 index bd14ba7..0000000 --- a/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/sqlalchemy-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/SKILL.md new file mode 100644 index 0000000..d09651d --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/SKILL.md @@ -0,0 +1,273 @@ +--- +name: sqlalchemy-azure-sql +description: >- + Uses SQLAlchemy correctly against Azure SQL Database, where the dialect appends an OUTPUT + clause to INSERT statements and that single clause explains two failures agents never connect: + a hard error on any table carrying a trigger, and fast_executemany appearing to do nothing. Use + when a user says "SQLAlchemy with Azure SQL", "mssql+pyodbc", "implicit_returning", + "fast_executemany", "insertmanyvalues", "Alembic against Azure SQL", or pastes "the target + table of the DML statement cannot have any enabled triggers if the statement contains an OUTPUT + clause without INTO clause". Also use when an ORM insert fails on one table only, or a bulk + load is no faster after fast_executemany was set. Covers the engine URL, the generated DML, + type mapping and Alembic. Driver choice is connect-from-python and retry connect-to-azure-sql. +--- + +# SQLAlchemy on Azure SQL Database + +SQLAlchemy reaches Azure SQL Database through a dialect whose generated SQL differs from every other +backend in one specific way, and almost every surprise here traces back to it. Every Python block +below was verified on 2026-09-03 against SQLAlchemy 2.0.52 and pyodbc 5.3.0 and its output quoted +verbatim; the version boundary below was measured the same day against 2.0.8, 2.0.9 and 2.0.10. +Engine error numbers are from Microsoft Learn. + +The dialect is named `mssql` and that name belongs only in the URL scheme. Everywhere else the +product is **Azure SQL Database**, and an agent that adopts the dialect name has begun giving advice +for a different product with different limits. + +## The engine URL + +```python +import os +from sqlalchemy import create_engine +from sqlalchemy.engine import URL + +credential = { # read them, never inline them + "username": os.environ["SQL_USER"], + "password": os.environ["SQL_PASSWORD"], +} +url = URL.create( + "mssql+pyodbc", + host=os.environ["SQL_HOST"], # .database.windows.net + port=1433, + database=os.environ["SQL_DATABASE"], + query={"driver": "ODBC Driver 18 for SQL Server"}, + **credential, +) +engine = create_engine(url, pool_pre_ping=True) +print(url.render_as_string(hide_password=True)) +``` + +`URL.create` escapes the characters that break a hand built URL; `render_as_string` logs one with +the password back as `***`. + +Driver choice, retry and passwordless connections belong to other skills; see References. + +## The OUTPUT clause, which is the whole skill + +To learn a server generated key the dialect adds an OUTPUT clause to the INSERT. Do not take that +on trust, compile it: + +```python +from sqlalchemy import Column, Integer, String, insert +from sqlalchemy.dialects import mssql +from sqlalchemy.orm import DeclarativeBase + +class Base(DeclarativeBase): # columns shared by both models + id = Column(Integer, primary_key=True) + sku = Column(String(20)) + +class Order(Base): + __tablename__ = "orders" + +class Audited(Base): # this one carries a trigger + __tablename__ = "audited_orders" + __table_args__ = {"implicit_returning": False} # so: no OUTPUT clause + +d = mssql.dialect() +print(insert(Order).values(sku="x").compile(dialect=d)) +print(insert(Audited).values(sku="x").compile(dialect=d)) +print(insert(Audited).values(sku="x").returning(Audited.id).compile(dialect=d)) +``` + +Three lines out, and every claim below is one: + +``` +INSERT INTO orders (sku) OUTPUT inserted.id VALUES (:sku) +INSERT INTO audited_orders (sku) VALUES (:sku) +INSERT INTO audited_orders (sku) OUTPUT inserted.id VALUES (:sku) +``` + +Azure SQL Database refuses the first when the target carries an enabled trigger: + +``` +Msg 334, Level 15, State 1 +The target table 'dbo.audited_orders' of the DML statement cannot have any enabled triggers +if the statement contains an OUTPUT clause without INTO clause. +``` + +Four properties decide how this plays out, and each is where agents guess wrong. + +1. **It is per table, and nothing detects it.** The dialect has no trigger reflection and does not + probe for one, so a codebase where 40 models work and one fails is the normal shape of the bug. +2. **The switch is on the table, not the engine**, as `__table_args__` above or + `implicit_returning=False` passed to `Table` in Core. The key then comes back from + `scope_identity()`, one extra round trip that works on a triggered table. Set it per table, not + everywhere as a precaution: it costs the batched insert path on tables that never needed it. +3. **The engine level parameter is a trap.** `create_engine(url, implicit_returning=True)` and the + `False` form are both accepted, both raise `SADeprecationWarning`, and neither changes one + character of the SQL above. +4. **The flag does not cover an explicit `returning()`.** That is the third printed line: with the + flag off, a hand written `insert().returning(...)` still compiles to OUTPUT without INTO, and + `update().returning(...)` and `delete().returning(...)` do the same. + +Triggers cluster on auditing and history tables, exactly where an ORM meets an existing schema. +Find them before mapping one, with the query under Check it worked. + +## fast_executemany, and what changed in 2.0 + +**Same cause as the trigger error above: the generated returning clause.** An ORM insert of a +mapped class with a server generated key carries one, because that is the OUTPUT clause, and that +is the case the flag does not reach, so by default it changes nothing for exactly the bulk +inserts people set it for. + +`fast_executemany=True` has existed since SQLAlchemy 1.3; what changed is what it does, and the +boundary is measurable with no database: + +```bash +python3 -c "import sqlalchemy; from sqlalchemy.dialects import mssql; \ +print(sqlalchemy.__version__, mssql.dialect().use_insertmanyvalues)" +``` + +That prints `2.0.8 True`, `2.0.9 False`, `2.0.10 True`. SQLAlchemy 2.0 introduced +**insertmanyvalues**, a batched INSERT form that returns keys, and because it returns keys it took +precedence over the driver's array binding and the flag stopped having any effect. 2.0.9 gave that +effect back for multi-parameter INSERT statements **carrying no returning clause**, switching +insertmanyvalues off here entirely to do it; 2.0.10 restored it, so **2.0.10 is the floor**. + +So the flag bites only where the INSERT carries no returning clause: the rows already hold their +keys, or the table is declared `implicit_returning=False`, or the load goes through Core rather +than tracked ORM objects. + +Documented costs make this a decision, not a default: the batch must **fit in memory**, the +parameter is honoured for the Microsoft ODBC driver only, and `setinputsizes` is **not used** +there, which is where the surprising type handling on large loads comes from. + +## Type mapping, before the first migration + +```python +from sqlalchemy import Column, MetaData, String, Table, Text, Unicode, UnicodeText +from sqlalchemy.dialects import mssql +from sqlalchemy.schema import CreateTable + +t = Table("t", MetaData(), + Column("a", String(50)), Column("b", Unicode(50)), + Column("c", Text()), Column("d", UnicodeText())) +print(CreateTable(t).compile(dialect=mssql.dialect())) # not connected +print(CreateTable(t).compile(dialect=mssql.dialect(deprecate_large_types=True))) # connected +``` + +The second print is what the ORM creates: + +``` +a VARCHAR(50) NULL, b NVARCHAR(50) NULL, c VARCHAR(max) NULL, d NVARCHAR(max) NULL +``` + +`String` is **not** Unicode. A model that uses it for names, addresses or user text gets a +non-Unicode column, and the mismatch between it and the Unicode parameter Python binds is the +implicit conversion that stops an index being used. Prefer `Unicode` or `NVARCHAR(n)` explicitly. + +That the two prints differ is the second trap. Offline, `deprecate_large_types` is `None` and `c` +and `d` compile to `TEXT` and `NTEXT`; on first connect the dialect sets it from the server version, +and Azure SQL Database is always past the threshold. Compiling offline is the normal way to inspect +DDL and wrong for exactly this pair, so never report a `TEXT` or `NTEXT` read out of a compiled +statement as what will be created. + +One default worth pinning: `Sequence("s")` with no `start` compiles to a bare +`CREATE SEQUENCE s`, and Microsoft Learn gives an ascending sequence's default start as the minimum +value of its type, `-9223372036854775808` for the default `bigint`. Pass `start=1`, and prefer an +identity column for surrogate keys. + +## Alembic + +Alembic drives the same dialect, so all the above applies to the DDL it emits. Read that DDL +before it runs, which is the point of offline mode: + +```bash +alembic revision --autogenerate -m "add audited_orders" +alembic upgrade head --sql > migration.sql # writes SQL, runs nothing +grep -nE 'TEXT|NTEXT|VARCHAR\(' migration.sql # the type mapping above, before it ships +alembic upgrade head +``` + +Three operations need arguments here that they need nowhere else, and autogenerate will not add +them: + +- `alter_column` requires `existing_type` when changing nullability. +- `drop_index` requires `table_name`. +- `drop_column` on a column with a DEFAULT, CHECK or single foreign key constraint takes + `mssql_drop_default`, `mssql_drop_check` or `mssql_drop_foreign_key` to drop the unnamed + constraint first. + +These come from the Alembic documentation, not from a run: Alembic was not installed on the machine +that produced the numbers above. Whether a migration is safe to run at all, +expand and contract, and never migrating on startup, are `schema-migrations-safely`. + +## Check it worked + +Three checks, in the order they catch things. **Which tables carry an enabled trigger** is first, +because nothing in Python will tell you: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -b -m-1 -Q \ + "SELECT OBJECT_SCHEMA_NAME(parent_id) + '.' + OBJECT_NAME(parent_id) AS triggered_table + FROM sys.triggers WHERE parent_class = 1 AND is_disabled = 0 ORDER BY 1;" +``` + +Expect one row per table needing `implicit_returning=False`, none on a schema with no triggers. +Keep `-m-1`: a severity 10 message otherwise prints with no `Msg` number, and `-b` will +not fail on one. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build a severity 10 message prints with no number whether you keep the flag or not. `build-app-on-azure-sql` tells the two +builds apart in one table. + +**Every model that needs the flag has it.** Print the other list and compare: + +```python +print(sorted(t.name for t in Base.metadata.tables.values() if t.implicit_returning)) +``` + +`Table.implicit_returning` is `True` by default and `False` once set, so expect no name from the +`sqlcmd` output here. Any that appears is the next `Msg 334`. + +**The columns the engine actually created.** Run this against the migrated database: + +```sql +SELECT OBJECT_NAME(c.object_id) AS tbl, c.name AS col, t.name AS type_name +FROM sys.columns AS c JOIN sys.types AS t ON t.user_type_id = c.user_type_id +WHERE OBJECTPROPERTY(c.object_id, 'IsUserTable') = 1 + AND t.name IN ('varchar', 'char', 'text', 'ntext'); +``` + +Expect zero rows anywhere text can hold a non-ASCII character. A `varchar` or `char` row is a +`String` column, a `text` or `ntext` row is DDL from a dialect that never connected, and the engine +accepts both without complaint, which is why nothing else catches them. + +## Do not + +- Do not claim `fast_executemany=True` sped up an ORM bulk insert without first checking that the + statement carries no returning clause. On the default configuration it did not. +- Do not inline a credential in an engine URL, and do not call the product `mssql`. + +## References + +- [SQLAlchemy Microsoft SQL Server dialect](https://docs.sqlalchemy.org/en/20/dialects/mssql.html): + triggers, identity, sequences, fast_executemany, setinputsizes. Fetch it before asserting what the + dialect emits. +- [insertmanyvalues](https://docs.sqlalchemy.org/en/20/core/connections.html#engine-insertmanyvalues): + what the batched form generates and how to turn it off. Read it when a bulk load is slow or noisy. +- [The OUTPUT clause](https://learn.microsoft.com/sql/t-sql/queries/output-clause-transact-sql): the + restriction in the engine's own words, per DML action. Read it to confirm the rule, not the text. +- [Alembic operations reference](https://alembic.sqlalchemy.org/en/latest/ops.html): the + dialect-specific arguments above. Read it while writing a migration that alters or drops a column. +- `connect-from-python`: driver choice, installation, pooling. Open it before writing the URL; the + first-party driver has a dialect only from SQLAlchemy 2.1.0b2, not for production. +- `connect-to-azure-sql`: encryption, retry, pool sizing. Open it the moment a transient error + appears: `pool_pre_ping` tests a connection at checkout and is not a retry policy. +- `entra-id-auth`: passwordless connections, which carry no user, no password and no + `Trusted_Connection`. Open it before putting a token flow near the engine URL. +- `design-azure-sql-schema`: key length, sizing, collation. Open it before the first migration. +- `schema-migrations-safely`: the migration doctrine Alembic inherits, including why a pipeline can + report success on a migration that only warned. diff --git a/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..6c6c481 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/sqlalchemy-azure-sql/skill.spec.jsonc @@ -0,0 +1,155 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "sqlalchemy-azure-sql", + "domain": "schema-and-orm", + "value": ["corrects-a-wrong-assumption", "not-in-training-data"], + "correction": "Agents treat SQLAlchemy against Azure SQL Database as the same job as SQLAlchemy against PostgreSQL, when the dialect silently appends OUTPUT inserted to every INSERT into a table whose key the server generates; that clause is rejected outright on any table carrying an enabled trigger, the dialect has no trigger detection and is not going to grow one, the only switch is implicit_returning=False declared per table by hand, and the same clause is why setting fast_executemany=True on ORM inserts appears to do nothing at all.", + "value_probe": { + "task": "We use SQLAlchemy 2.0 with the ORM against a database in Azure. Everything works except one table.\n\nThe `orders` table has an audit trigger on it that the data team added. A plain `session.add(Order(...))` followed by `session.commit()` on that table fails, and the message says the target table of the statement cannot have any enabled triggers when the statement contains an OUTPUT clause without an INTO clause. The identical pattern against every other mapped class is fine. We are not writing any raw SQL here and nothing in our code mentions OUTPUT.\n\nSeparately, and possibly unrelated, we turned on the driver's fast execute option on the engine to speed up a nightly load that inserts a few hundred thousand rows through the ORM. It made no measurable difference at all, and we cannot see the batching we expected in the traces.\n\nWhat is causing the first problem, what is the fix, and is the second one connected?", + "gate": "The answer identifies that the dialect itself adds the returning clause to inserts into tables whose keys the server generates, and turns that behaviour off for the affected table explicitly, rather than dropping the trigger, rewriting the insert by hand, or changing the driver.", + "states": [ + "The clause is added by the dialect, not by the application code, on inserts into a table whose primary key value the server generates.", + "The dialect does not detect triggers on the target table and will not do so, so the behaviour has to be turned off explicitly by the author.", + "The switch is declared per table by hand, so every affected table has to be found and marked.", + "The two problems have the same cause, because the same generated clause prevents the ORM inserts from being sent as one batched execution." + ], + "refutes": [ + "SQLAlchemy does not emit a returning clause unless the application asks for one, so the clause must be coming from a trigger, from raw SQL, or from somewhere else in the application.", + "The dialect detects an enabled trigger on the target table and adjusts the generated statement automatically, so an upgrade will resolve this.", + "The right fix is to drop or disable the audit trigger, or to replace it with something outside the database.", + "The fast execute option having no effect on these ORM inserts is a driver, batch size, or connection tuning problem to be solved separately from the first issue." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "connect-from-python", + "connect-to-azure-sql", + "entra-id-auth", + "design-azure-sql-schema", + "schema-migrations-safely", + "ef-core-azure-sql", + "prisma-azure-sql", + "django-azure-sql", + "build-app-on-azure-sql" + ], + "validation": { + "target": "both", + "setup": [ + "SQLAlchemy 2.0.10 or later and a driver the dialect accepts", + "a reachable Azure SQL database, or the local container", + "at least one table carrying an enabled trigger" + ], + "assert": [ + "every mapped table that carries a trigger declares implicit_returning=False", + "no explicit returning() call targets a table that carries a trigger", + "fast_executemany is only claimed as a speedup for inserts that carry no returning clause", + "SQLAlchemy is pinned at 2.0.10 or later wherever fast_executemany is set", + "columns that must hold non-ASCII text are declared Unicode rather than String", + "no credential appears in the engine URL in source" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_sa_trig (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, sku nvarchar(20) NOT NULL);\nGO\nCREATE TRIGGER dbo.trg_probe_sa_trig ON dbo.probe_sa_trig AFTER INSERT AS SET NOCOUNT ON;\nGO\nINSERT INTO dbo.probe_sa_trig (sku) OUTPUT inserted.id VALUES (N'x');", + "expect": { + "error": 334 + }, + "why": "The whole skill, made executable. This is character for character what SQLAlchemy 2.0.52 compiles for a mapped class with a server generated key, measured on 2026-09-03 as INSERT INTO orders (sku) OUTPUT inserted.id VALUES (:sku), run against a table that carries a trigger. Microsoft Learn lists error 334 at severity 15 on the Database Engine events and errors page, which is also the correction this rewrite made: the skill previously printed Level 16." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_scope (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, sku nvarchar(20) NOT NULL);\nGO\nCREATE TRIGGER dbo.trg_probe_sa_scope ON dbo.probe_sa_scope AFTER INSERT AS SET NOCOUNT ON;\nGO\nINSERT INTO dbo.probe_sa_scope (sku) VALUES (N'x');\nSELECT CAST(SCOPE_IDENTITY() AS int) AS new_id;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The fix, asserted rather than described. implicit_returning=False makes the dialect compile the INSERT with no OUTPUT clause and read the key back with scope_identity() instead, which is exactly this pair of statements. The same table that refuses the probe above accepts this one and still yields the generated key, so the flag costs a round trip and nothing else." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_into (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, sku nvarchar(20) NOT NULL);\nGO\nCREATE TRIGGER dbo.trg_probe_sa_into ON dbo.probe_sa_into AFTER INSERT AS SET NOCOUNT ON;\nGO\nDECLARE @out TABLE (id int);\nINSERT INTO dbo.probe_sa_into (sku) OUTPUT inserted.id INTO @out VALUES (N'x');\nSELECT id FROM @out;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "Proves the last four words of the message are load bearing: the restriction is on OUTPUT without INTO, not on OUTPUT. An agent that reads error 334 as \"this table cannot use OUTPUT\" writes off a working option. The dialect does not generate this form, which is why implicit_returning=False and not an INTO rewrite is the answer in SQLAlchemy, but a reader hand writing DML against a triggered table needs to know it exists." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_upd (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, sku nvarchar(20) NOT NULL);\nINSERT INTO dbo.probe_sa_upd (sku) VALUES (N'x');\nGO\nCREATE TRIGGER dbo.trg_probe_sa_upd ON dbo.probe_sa_upd AFTER UPDATE AS SET NOCOUNT ON;\nGO\nUPDATE dbo.probe_sa_upd SET sku = N'y' OUTPUT inserted.id;", + "expect": { + "error": 334 + }, + "why": "The skill's fourth property, which implicit_returning=False does not cover. Compiled on 2026-09-03 against SQLAlchemy 2.0.52, update().returning(Audited.id) on a table declared with the flag off still emits UPDATE audited_orders SET sku=:sku OUTPUT inserted.id, and this is that statement. Microsoft Learn scopes the restriction per DML action, so an UPDATE trigger is what makes an UPDATE with OUTPUT fail." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_find (id int IDENTITY(1,1) NOT NULL PRIMARY KEY, sku nvarchar(20) NOT NULL);\nGO\nCREATE TRIGGER dbo.trg_probe_sa_find ON dbo.probe_sa_find AFTER INSERT AS SET NOCOUNT ON;\nGO\nSELECT OBJECT_SCHEMA_NAME(parent_id) + '.' + OBJECT_NAME(parent_id) AS triggered_table\nFROM sys.triggers\nWHERE parent_class = 1 AND is_disabled = 0 AND OBJECT_NAME(parent_id) = 'probe_sa_find';", + "expect": { + "rows": [ + [ + "dbo.probe_sa_find" + ] + ] + }, + "why": "The first check in the skill's Check it worked section, run against a table that needs the flag. The dialect has no trigger reflection, so this catalog query is the only way to build the list of tables that must declare implicit_returning=False. It is filtered to one table name rather than listing every trigger in the scratch database, because the other probes here create triggers of their own and no probe should depend on the order they ran in." + }, + { + "sql": "CREATE SEQUENCE dbo.probe_sa_seq;\nGO\nDECLARE @v bigint = NEXT VALUE FOR dbo.probe_sa_seq;\nSELECT CASE WHEN @v = -9223372036854775808 THEN 1 ELSE 0 END AS starts_at_bigint_minimum;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "SQLAlchemy compiles Sequence(\"s\") with no start to exactly this bare CREATE SEQUENCE, measured on 2026-09-03 against 2.0.52, and Microsoft Learn states that a sequence with no data type defaults to bigint and one with no START WITH defaults to the minimum value of its type. So an ORM model that declares a Sequence and forgets start=1 gets keys beginning at the bottom of the bigint range. The comparison runs in the engine and returns 1 or 0, because -9223372036854775808 is past the range a JSON number carries exactly." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_varchar (a varchar(50) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, b nvarchar(50) NULL);\nGO\nDECLARE @t nvarchar(50) = NCHAR(0x6F22) + NCHAR(0x5B57);\nINSERT INTO dbo.probe_sa_varchar (a, b) VALUES (@t, @t);\nSELECT CASE WHEN a = b THEN 1 ELSE 0 END AS varchar_kept_the_text FROM dbo.probe_sa_varchar;", + "expect": { + "rows": [ + [ + 0 + ] + ] + }, + "why": "What String(50) costs. The dialect compiles String(50) to VARCHAR(50) and Unicode(50) to NVARCHAR(50), measured on 2026-09-03, and this writes the same two characters into both. They come back different, with no error and no warning at insert time, which is why a model that used String for user text is usually found much later. The collation is pinned on the column rather than inherited from the database so the result does not depend on how the scratch database was created, and the characters are built with NCHAR rather than written as literals so it does not depend on the client encoding either." + }, + { + "sql": "CREATE TABLE dbo.probe_sa_text (c text NULL, d ntext NULL);", + "expect": { + "succeeds": true + }, + "why": "The reason the skill's third check queries the catalog rather than trusting a deployment to fail. DDL generated by a dialect that never opened a connection maps Text() to TEXT and UnicodeText() to NTEXT, and the engine takes both without complaint, so nothing between the model and production reports the problem. If this ever starts failing, the offline compilation trap stops being silent and that part of the skill needs rewriting." + } + ] + }, + "triggering": { + "explicit": [ + "Use the sqlalchemy-azure-sql skill", + "Wire SQLAlchemy up to Azure SQL Database", + "Set up SQLAlchemy models and an engine against Azure SQL", + "Should I turn on fast_executemany for my Azure SQL bulk load" + ], + "implicit": [ + "the target table of the DML statement cannot have any enabled triggers if the statement contains an OUTPUT clause without INTO clause", + "Msg 334 when my ORM inserts a row", + "I set fast_executemany True and the bulk insert is exactly as slow as before", + "session.add then commit blows up on one table only and I cannot see why", + "my Alembic migration cannot change a column to NOT NULL on Azure SQL" + ], + "negative": [ + "install the ODBC driver so my Python container can reach Azure SQL", + "which Python driver should I use, pyodbc or the Microsoft one", + "my Prisma schema will not accept a Json field", + "configure EnableRetryOnFailure for EF Core against Azure SQL", + "add retry with backoff for transient errors on Azure SQL", + "design the primary key and index strategy for a new table", + "set up Django models against Azure SQL" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-correctness b/plugins/microsoft-sql-vscode/skills/t-sql-correctness deleted file mode 120000 index 883f308..0000000 --- a/plugins/microsoft-sql-vscode/skills/t-sql-correctness +++ /dev/null @@ -1 +0,0 @@ -../../../skills/t-sql-correctness \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-correctness/SKILL.md b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/SKILL.md new file mode 100644 index 0000000..b21abf9 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/SKILL.md @@ -0,0 +1,224 @@ +--- +name: t-sql-correctness +description: >- + Writes T-SQL that returns the right answer on Azure SQL Database, and catches statements that + return a wrong answer with no error: NULL compared using = or <> or NOT IN, ISNULL and COALESCE + differing in return type, integer division truncating, a string variable declared with no + length, and a session where QUOTED_IDENTIFIER is OFF. Also corrects PostgreSQL and MySQL habit + (LIMIT, RETURNING, SERIAL, ILIKE, NOW(), true, false, double-quoted literals, TEXT columns, ON + CONFLICT, USE) and the opposite mistake of avoiding syntax supported since 2025. Use when + writing, porting or reviewing SQL, and for "why is that row missing", "why did NOT IN return + nothing", "why is this average wrong", "how do I paginate", "how do I get the id I just + inserted", "is this comparison case sensitive". Upserts are t-sql-upserts-merge, JSON + t-sql-json-and-openjson, table design design-azure-sql-schema. +--- + +# Write T-SQL that returns the right answer + +Three failures. **The silent one first**, because nothing raises it: the statement succeeds, the +row count looks plausible, the answer is wrong. Then **PostgreSQL habit**, which at least fails +loudly. Then **overcorrection**, hand-rolling what the engine now does natively. + +Checked against Microsoft Learn and sqlcmd 1.10.0, 2026-09-03. + +## Wrong answers that raise no error + +**Nothing equals NULL, and nothing is unequal to it either.** `ANSI_NULLS` is permanently `ON` +and `SET ANSI_NULLS OFF` is deprecated, so every comparison against `NULL` is `UNKNOWN`, and +`WHERE` keeps only the rows that are `TRUE`. + +```sql +CREATE TABLE dbo.orders (id INT, status NVARCHAR(20) NULL); +INSERT INTO dbo.orders VALUES (1, N'shipped'), (2, N'held'), (3, NULL); + +DECLARE @p NVARCHAR(20) = NULL; +SELECT COUNT(*) FROM dbo.orders WHERE status = @p; -- 0, not 1 +SELECT COUNT(*) FROM dbo.orders WHERE status <> N'shipped'; -- 1, not 2 + +-- Null-safe equality, true or false and never unknown: +SELECT COUNT(*) FROM dbo.orders WHERE status IS NOT DISTINCT FROM @p; -- 1 +``` + +The second line reaches production: "everything not shipped" drops every row whose status is +unknown. + +**`NOT IN` over a nullable column returns nothing at all.** One `NULL` in the subquery makes the +predicate `UNKNOWN` for every candidate row, so the answer is empty rather than short by one. + +```sql +CREATE TABLE dbo.assigned (customer_id INT NULL); +INSERT INTO dbo.assigned VALUES (1), (NULL); +CREATE TABLE dbo.customers (id INT NOT NULL); +INSERT INTO dbo.customers VALUES (1), (2), (3); + +SELECT COUNT(*) FROM dbo.customers c +WHERE c.id NOT IN (SELECT customer_id FROM dbo.assigned); -- 0 + +-- NOT EXISTS is the fix, and the habit worth defaulting to: +SELECT COUNT(*) FROM dbo.customers c +WHERE NOT EXISTS (SELECT 1 FROM dbo.assigned a WHERE a.customer_id = c.id); -- 2 +``` + +**Aggregates skip nulls, and say nothing about it.** Measured 2026-09-03 on `EngineEdition` 5 +through the **ODBC `sqlcmd`** at `-I -m-1`, the build that does print a severity 10 message's `Msg` +number: this batch returns 30 and **no message at all**. Which build was used is the whole weight of +that sentence, because go-sqlcmd 1.10.0 prints no `Msg` header on a severity 10 message at any `-m` +value, so a silent run there would prove nothing. Message 8153, "Null value is eliminated by an +aggregate", is in `sys.messages` at severity 10 and was never raised. Microsoft Learn's +`SET ANSI_WARNINGS` page says a warning is generated. It is not, so do not plan on being told. + +```sql +CREATE TABLE dbo.readings (v INT NULL); +INSERT INTO dbo.readings VALUES (10), (NULL), (20); +SET ANSI_WARNINGS ON; -- makes no difference here +SELECT SUM(v), COUNT(*), COUNT(v), AVG(v) FROM dbo.readings; -- 30, 3, 2, 15 +``` + +`AVG` divides by 2, not 3, and `COUNT(v)` disagrees with `COUNT(*)` for the same reason. If the +intent was 10, write `AVG(ISNULL(v, 0))` and mean it. + +**Integer division truncates, and `ISNULL` truncates its own replacement.** `/` returns the +higher-precedence operand's type, and two integers give an integer. `ISNULL` returns the type of +its **first** argument and converts the second into it; `COALESCE` returns the highest-precedence +type of all of them. + +```sql +SELECT 7 / 2, CAST(7.0 / 2 AS DECIMAL(4,1)); -- 3 and 3.5 + +DECLARE @short NVARCHAR(3) = NULL; +SELECT ISNULL(@short, N'abcdef'), COALESCE(@short, N'abcdef'); -- abc and abcdef +``` + +`ISNULL` also reports its result as not nullable where `COALESCE` reports nullable, so a computed +column that must come out `NOT NULL` needs `ISNULL`. + +**A string with no length is one character, except in a cast, where it is thirty.** Assigning to +an undersized variable truncates in silence; only an undersized column refuses. + +```sql +DECLARE @v NVARCHAR = N'abcdef'; +SELECT @v, LEN(CAST(REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 40) AS NVARCHAR)); -- a and 30 +``` + +Always give a length. The inner cast is there because `REPLICATE` otherwise returns +`nvarchar(4000)`. + +## Session settings change what a statement means + +`QUOTED_IDENTIFIER` is decided at parse time, so the same text is two different statements: + +```sql +SET QUOTED_IDENTIFIER OFF; +SELECT "not a column"; -- returns the string, one row +SET QUOTED_IDENTIFIER ON; +SELECT "not a column"; -- Msg 207, invalid column name +``` + +`ON` is the default and the ODBC and OLE DB drivers set it on connect, so application code sees +`ON`. The container's own `sqlcmd` leaves it **OFF**, so a script behaves one way in a container +shell and another way run by the app. Pass `-I` to settle it. + +## PostgreSQL habit, translated + +| It will write | It must write | Because | +|---|---|---| +| `LIMIT 10 OFFSET 20` | `ORDER BY id OFFSET 20 ROWS FETCH NEXT 10 ROWS ONLY` | `OFFSET` and `FETCH` are clauses **of** `ORDER BY`; `FETCH` without `OFFSET` is invalid | +| `RETURNING id` | `OUTPUT INSERTED.id` | Read the trigger rule below before shipping it | +| `id SERIAL PRIMARY KEY` | `id INT IDENTITY(1,1) PRIMARY KEY` | Or a `SEQUENCE` when the generator is shared | +| `is_active = true`, `WHERE is_active` | `is_active = 1` | No Boolean type. `bit` is an integer, and a value rather than a predicate | +| `NOW()` | `SYSDATETIME()`, or `SYSUTCDATETIME()` | `CURRENT_TIMESTAMP` also works and is ANSI | +| `name ILIKE 'ana%'` | `name LIKE 'ana%'` | The default collation `SQL_Latin1_General_CP1_CI_AS` is already case-insensitive | +| `WHERE name = "ana"` | `WHERE name = 'ana'` | Escape a quote by doubling it, `'it''s'`, and prefix Unicode with `N` | +| `bio TEXT` | `bio NVARCHAR(MAX)` | `text` and `ntext` are deprecated and barred from several operators | +| `ON CONFLICT DO UPDATE` | See `t-sql-upserts-merge` | That skill owns upserts, including when not to use `MERGE` | +| `USE otherdb;` | Open a new connection | Unsupported, along with cross-database three and four part names | + +`USE` is the trap that survives review: it **works on the container** and fails in the cloud. Open [references/postgres-to-tsql.md](references/postgres-to-tsql.md) before porting a +schema or a query file, and when a statement crosses a database boundary. + +`TOP` and `OFFSET FETCH` cannot be combined in one query expression, and a stable page needs a +**unique** sort key: `ORDER BY placed_at DESC, order_id DESC`. + +A bare `OUTPUT` fails with `Msg 334` when the target has an enabled trigger for that action, so +capture into a table variable: + +```sql +CREATE TABLE dbo.new_orders (order_id INT IDENTITY PRIMARY KEY, total DECIMAL(9,2)); +DECLARE @new TABLE (order_id INT); +INSERT INTO dbo.new_orders (total) +OUTPUT INSERTED.order_id INTO @new +VALUES (19.99); +SELECT order_id FROM @new; +``` + +The `INTO` target has its own limits: no enabled triggers, neither side of a foreign key +(`Msg 332`), no enabled `CHECK` constraints or rules (`Msg 333`), and no guaranteed row order. +Prefer `OUTPUT` to `SCOPE_IDENTITY()`: it returns every row of a multi-row insert, and works on +`UPDATE`, `DELETE` and `MERGE`. + +## Do not avoid these. They work + +| Available | Note | +|---|---| +| `a \|\| b` and `\|\|=`, `UNISTR` | Generally available July 2025. Unlike `CONCAT`, `\|\|` yields `NULL` if any input is `NULL` | +| Regular expression functions | Generally available November 2025. Route to `t-sql-regex-and-new-functions` | +| `STRING_AGG(x, ',') WITHIN GROUP (ORDER BY x)` | Any compatibility level, but returns `nvarchar(4000)` for `nvarchar(1..4000)` input, so cast to `max` or lose the tail | +| `TRIM(BOTH '.' FROM s)` | The positional keywords need a recent compatibility level; below it they are a **parse** error | +| `GREATEST(a, b, c)`, `LEAST(...)` | Nulls ignored unless every argument is null | +| `a IS NOT DISTINCT FROM b` | Null-safe equality, as above | + +A migrated database can sit at an old compatibility level, where the failure looks exactly like +the function not existing: `Msg 102`, `Msg 195` or `Msg 208` at level 150. Check the level before +believing the error. See `post-migration-compatibility-level`. + +## Check it worked + +Save this as `check-correctness.sql`. The exit code proves nothing here: every statement +succeeds. + +```sql +-- 1. The session your script was actually parsed and run under. +SELECT IIF((256 & @@OPTIONS) = 256, 'ON', 'OFF') AS quoted_identifier, + IIF((32 & @@OPTIONS) = 32, 'ON', 'OFF') AS ansi_nulls, + IIF((8 & @@OPTIONS) = 8, 'ON', 'OFF') AS ansi_warnings; + +-- 2. String columns declared with no length: they hold one character. +SELECT OBJECT_NAME(object_id) AS t, name AS c FROM sys.columns +WHERE system_type_id IN (231, 239, 167, 175) AND max_length <= 2; +``` + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C -I -b -m-1 \ + -i check-correctness.sql -o check-correctness.out +``` + +Expected: check 1 returns `ON ON ON`, check 2 no rows. An `OFF` means your script was +parsed under different rules from the ones the application connects with. `-m-1` is there because +`-b` sets a non-zero exit only at severity 11 and above, so any severity 10 message leaves a +script reporting success. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build a quiet output file is not evidence that the engine +stayed quiet. `build-app-on-azure-sql` tells the two builds apart in one table. + +## Do not + +- Do not compare with `= NULL` or `<> NULL`, and do not "fix" it by turning `ANSI_NULLS` off. + That setting is deprecated and permanently `ON`. +- Do not use `NOT IN` against a subquery over a nullable column. Use `NOT EXISTS`. +- Do not swap `ISNULL` for `COALESCE` as a cosmetic edit. The return type differs, so the value + can change. +- Do not translate `ILIKE` into `LOWER(col) = LOWER(@v)`. It fixes nothing on a case-insensitive + collation and costs the index seek. +- Do not size, collate or index columns here. `design-azure-sql-schema` owns that, including why + a `json` column compared with `=` fails with `Msg 402`. +- Do not put error handling here. `t-sql-error-handling` owns `TRY`, `CATCH` and + `XACT_ABORT`, and `t-sql-programmability-objects` owns triggers. + +## References + +- [references/postgres-to-tsql.md](references/postgres-to-tsql.md): open it when porting a schema + or a query file, when a statement crosses a database boundary, or to find the Microsoft Learn + page behind a row in the table above. diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-correctness/references/postgres-to-tsql.md b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/references/postgres-to-tsql.md new file mode 100644 index 0000000..cdc2639 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/references/postgres-to-tsql.md @@ -0,0 +1,151 @@ +# PostgreSQL and MySQL habits, translated for Azure SQL Database + +Checked against Microsoft Learn on 2026-08-27. Where a claim is version-dependent, the date it +became true is given, because that is the part most likely to be out of date in training data. + +## Contents + +- [How to use this file](#how-to-use-this-file) +- [Statements and clauses](#statements-and-clauses) + - [The local container is the exception](#the-local-container-is-the-exception) +- [Operators and expressions](#operators-and-expressions) +- [Data types](#data-types) +- [Things that changed recently](#things-that-changed-recently) +- [Things that are still not there](#things-that-are-still-not-there) +- [Sources](#sources) + +## How to use this file + +Read the left column as the thing about to be written and the right column as the replacement. +Where a row has a note, the note is the part that bites after the substitution compiles. + +Two rows deliberately have no answer here, because another skill owns them: upserts go to +`t-sql-upserts-merge`, and anything JSON goes to `t-sql-json-and-openjson`. + +## Statements and clauses + +| Elsewhere | Azure SQL Database | Note | +|---|---|---| +| `LIMIT n` | `ORDER BY OFFSET 0 ROWS FETCH NEXT n ROWS ONLY`, or `TOP (n)` | `OFFSET` and `FETCH` are part of the `ORDER BY` clause, so a sort is mandatory | +| `LIMIT n OFFSET m` | `ORDER BY OFFSET m ROWS FETCH NEXT n ROWS ONLY` | `OFFSET` is required and `FETCH` is optional, never the other way round | +| `LIMIT` inside a view or subquery | `TOP` or `OFFSET FETCH` | `ORDER BY` is otherwise invalid in views, inline functions, derived tables and subqueries | +| `RETURNING ` | `OUTPUT INSERTED.` | Without `INTO`, the target cannot have an enabled trigger for that action | +| `RETURNING` on delete | `OUTPUT DELETED.` | `INSERTED` is unavailable on `DELETE`, `DELETED` on `INSERT` | +| `ON CONFLICT ... DO UPDATE` | See `t-sql-upserts-merge` | | +| `CREATE TABLE IF NOT EXISTS` | `IF OBJECT_ID(N'dbo.t', N'U') IS NULL BEGIN CREATE TABLE ... END` | `CREATE TABLE` has no `IF NOT EXISTS` clause. `DROP TABLE IF EXISTS` does exist, and `CREATE OR ALTER` exists for modules, not tables | +| `USE otherdb;` | Open a new connection to that database | Unsupported in Azure SQL Database. Cross-database three and four part names are unsupported too, except `tempdb` and the current database. **All of this works on the local Azure SQL Database container**, so a local run will not warn you. See [the local container is the exception](#the-local-container-is-the-exception) | +| `SELECT ... FOR UPDATE` | `SELECT ... WITH (UPDLOCK, ROWLOCK)` | A hint, not a clause | +| `ORDER BY x NULLS LAST` | `ORDER BY CASE WHEN x IS NULL THEN 1 ELSE 0 END, x` | There is no `NULLS FIRST` or `NULLS LAST`. Nulls sort as the lowest value | +| `DISTINCT ON (col)` | `ROW_NUMBER() OVER (PARTITION BY col ORDER BY ...)` filtered to 1 | | +| `information_schema` views | They exist, but `sys.` catalog views carry more | Prefer `sys.objects`, `sys.columns`, `sys.indexes` | + +### The local container is the exception + +This is the one row on the page where the local Azure SQL Database container and Azure SQL Database +in the cloud genuinely differ, and it differs in the direction that hurts. Measured against the +container: `USE appdb` returns `Changed database context to 'appdb'`, and `SELECT ... FROM +appdb.sys.objects` issued from `master` succeeds. Both fail against Azure SQL Database, where each +database is its own boundary. + +The container is a single engine hosting several databases, so it accepts both. A query that +crosses a database boundary therefore passes locally, ships, and fails in the cloud with nothing in +the local run to warn you. Test anything that crosses a database boundary against Azure SQL +Database, or do not write it. + +## Operators and expressions + +| Elsewhere | Azure SQL Database | Note | +|---|---|---| +| `a \|\| b` | `a \|\| b` works, and so do `a + b` and `CONCAT(a, b)` | Generally available since July 2025. `\|\|` and `+` propagate `NULL`; `CONCAT` treats `NULL` as an empty string | +| `x ILIKE 'a%'` | `x LIKE 'a%'` | The default collation is already case-insensitive | +| `LOWER(x) = LOWER(@v)` | `x = @v` | Only add `COLLATE` when the sensitivity genuinely has to differ from the column's | +| `true` / `false` | `1` / `0` | `bit` is an integer type taking `1`, `0` or `NULL`. The strings `'TRUE'` and `'FALSE'` convert to `1` and `0` | +| `WHERE flag` | `WHERE flag = 1` | `bit` is a value, not a predicate | +| `a IS DISTINCT FROM b` | `a IS DISTINCT FROM b` | Supported. Guarantees true or false even when an operand is `NULL` | +| `COALESCE` chains to emulate null-safe equality | `IS NOT DISTINCT FROM` | | +| `GREATEST(a, b)` / `LEAST(a, b)` | Same names, supported | Up to 254 arguments. `NULL` arguments are ignored unless all are `NULL` | +| `NOW()` | `SYSDATETIME()` | `SYSUTCDATETIME()` for UTC, `CURRENT_TIMESTAMP` for the ANSI spelling | +| `EXTRACT(YEAR FROM d)` | `DATEPART(year, d)` | | +| `x::int` | `CAST(x AS int)` | `CONVERT` when a style code is needed | +| `SUBSTRING(s FROM 1 FOR 3)` | `SUBSTRING(s, 1, 3)` | | +| `string_agg(x, ',')` | `STRING_AGG(x, ',') WITHIN GROUP (ORDER BY x)` | Available at any compatibility level. Nulls are skipped along with their separator | +| `trim(both '.' from s)` | `TRIM(BOTH '.' FROM s)` | The positional keywords are the newer part of the function. The documentation attaches a compatibility level 160 requirement to them | +| `'it\'s'` | `'it''s'` | Escape a single quote by doubling it. There is no backslash escape | +| `"col"` as a string | `'col'` | Double quotes are identifiers while `QUOTED_IDENTIFIER` is `ON`, which is the default and what the drivers set | +| `$1`, `$2` parameters | `@name` parameters | The wire placeholder is the driver's business; the T-SQL name is `@`-prefixed | + +## Data types + +| Elsewhere | Use here | Note | +|---|---|---| +| `TEXT`, `VARCHAR` without length | `NVARCHAR(MAX)`, or `NVARCHAR(n)` when a bound is known | `text` and `ntext` are excluded from `ORDER BY` and from the `\|\|` operator, which is a good signal of where they are heading | +| `BOOLEAN` | `bit` | | +| `SERIAL`, `BIGSERIAL`, `GENERATED ALWAYS AS IDENTITY` | `INT IDENTITY(1,1)`, `BIGINT IDENTITY(1,1)` | A `SEQUENCE` when the generator must be shared across tables | +| `UUID` | `uniqueidentifier` | `NEWSEQUENTIALID()` as a default is friendlier to a clustered index than `NEWID()` | +| `BYTEA` | `varbinary(max)` | | +| `TIMESTAMP` | `datetime2` | Note that the T-SQL type named `timestamp` is a row version, not a time | +| `TIMESTAMPTZ` | `datetimeoffset` | | +| `NUMERIC(p,s)` | `decimal(p,s)` | | +| `DOUBLE PRECISION` | `float` | | +| `JSON`, `JSONB` | See `t-sql-json-and-openjson` | That skill owns the storage choice and the query surface | +| Array types | No equivalent. Model as a child table | A delimited string plus `STRING_SPLIT` is a workaround, not a design | + +Two general notes. `nvarchar` stores UTF-16 and is the safe default for user text; a UTF-8 database +or column collation is available if byte size matters, and it applies to `char` and `varchar`, not +to `nchar` and `nvarchar`. And the character types have no implicit length: `NVARCHAR` written +without a length in a cast defaults to a single character in some contexts, so always give the +length. + +## Things that changed recently + +Anything here was different not long ago, so check rather than recall. + +| What | Since | Detail | +|---|---|---| +| `\|\|` string concatenation and `\|\|=` compound assignment | July 2025 | ANSI concatenation. Excludes the `xml`, `json`, `image`, `ntext` and `text` types. Result over 8,000 bytes truncates unless one operand is a large value type | +| `UNISTR` for Unicode string literals | July 2025 | | +| Regular expression functions | November 2025 | Seven functions. Three of them require a compatibility level check first, which is `t-sql-regex-and-new-functions` | +| `DATEADD` accepting a `bigint` number | November 2025 | | +| Vector data type and vector functions | June 2025 | Not this skill's subject, but a common source of "that cannot exist" | + +## Things that are still not there + +Verified absent, so an agent should stop looking rather than invent a spelling. + +- `LIMIT` in any form. +- `CREATE TABLE ... IF NOT EXISTS`. Guard with `IF OBJECT_ID(...) IS NULL`. +- `USE` to switch database context, and cross-database queries by three or four part name. Absent + in Azure SQL Database only. Both work on the local Azure SQL Database container, so this is the + one entry on the list a local run will not confirm for you: test it against the cloud. +- A Boolean data type. +- `NULLS FIRST` and `NULLS LAST`. +- Array and composite column types. + +## Sources + +All Microsoft Learn, all read on 2026-08-27. + +- T-SQL differences: `/azure/azure-sql/database/transact-sql-tsql-differences-sql-server` + (`USE` unsupported, cross-database names, the unsupported statement list) +- ORDER BY clause: `/sql/t-sql/queries/select-order-by-clause-transact-sql` (`OFFSET FETCH` syntax, + `TOP` cannot be combined with it, stable paging conditions, nulls sort lowest) +- OUTPUT clause: `/sql/t-sql/queries/output-clause-transact-sql` (the enabled-trigger restriction, + the `INTO` target restrictions, `INSERTED` and `DELETED` availability) +- Double pipe operator: `/sql/t-sql/language-elements/string-concatenation-pipes-transact-sql` + (applies to Azure SQL Database, null behaviour, truncation, excluded types) +- What is new in Azure SQL Database: + `/azure/azure-sql/database/doc-changes-updates-release-notes-whats-new` (the general availability + dates quoted above) +- CREATE DATABASE: `/sql/t-sql/statements/create-database-transact-sql` (the default collation + `SQL_Latin1_General_CP1_CI_AS`, and `CATALOG_COLLATION` fixed at creation) +- Collation and Unicode support: `/sql/relational-databases/collations/collation-and-unicode-support` + (per-database data and catalog collations, UTF-8 support) +- SET QUOTED_IDENTIFIER: `/sql/t-sql/statements/set-quoted-identifier-transact-sql` (`ON` is the + default, double quotes delimit identifiers, brackets are unaffected) +- bit: `/sql/t-sql/data-types/bit-transact-sql` +- STRING_AGG: `/sql/t-sql/functions/string-agg-transact-sql` (return type table, nulls skipped, any + compatibility level) +- TRIM: `/sql/t-sql/functions/trim-transact-sql` +- GREATEST: `/sql/t-sql/functions/logical-functions-greatest-transact-sql` +- IS [NOT] DISTINCT FROM: `/sql/t-sql/queries/is-distinct-from-transact-sql` +- CREATE TABLE: `/sql/t-sql/statements/create-table-transact-sql` (no `IF NOT EXISTS` in the syntax) diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-correctness/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/skill.spec.jsonc new file mode 100644 index 0000000..1590a40 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-correctness/skill.spec.jsonc @@ -0,0 +1,253 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "t-sql-correctness", + "domain": "tsql-correctness-and-patterns", + "value": ["corrects-a-wrong-assumption"], + "correction": "Asked for SQL against Azure SQL Database, an agent writes statements that succeed and return the wrong answer. Comparing a nullable column with = or <> or NOT IN is three-valued logic, not two: WHERE status = @p finds nothing when @p is NULL, WHERE status <> 'shipped' silently drops every row whose status is NULL, and NOT IN over a subquery holding one NULL returns the empty set rather than a set short by one. Aggregates skip nulls, so AVG divides by the non-null count and COUNT(v) disagrees with COUNT(*), and measured on 2026-09-03 against EngineEdition 5 through sqlcmd -I -m-1 the engine says nothing at all about it: message 8153 is defined in sys.messages at severity 10 and was never raised, while Microsoft Learn's SET ANSI_WARNINGS page states a warning is generated. Integer division truncates because / returns the higher-precedence operand's type. ISNULL returns the type of its FIRST argument and converts the second into it, so ISNULL over an NVARCHAR(3) silently returns 'abc' where COALESCE returns 'abcdef'. A string declared with no length is one character, and thirty in a cast. And QUOTED_IDENTIFIER is decided at parse time, so the same double-quoted text is a string literal in one session and an identifier in another: the ODBC and OLE DB drivers set it ON at connect, the container's own sqlcmd leaves it OFF. The second failure is the loud one, PostgreSQL and MySQL syntax under a T-SQL name, and the third is overcorrecting into syntax a decade old when the engine has shipped the ANSI double pipe operator, IS NOT DISTINCT FROM, GREATEST and STRING_AGG.", + "value_probe": { + "gate": "The answer explains all three wrong results as consequences of how the engine evaluates comparisons and aggregates when a column holds no value, and returns rewritten queries that produce the intended numbers. An answer that blames the data or the application, or that corrects only one of the three, has not reached it.", + "task": "Our order dashboard on Azure SQL Database disagrees with itself and nobody is getting an error message. dbo.orders has id, customer_id, status and discount_pct, and dbo.cancelled_orders has order_id. Three things are wrong.\n\nThe \"not shipped\" tile runs:\n\n```sql\nSELECT COUNT(*) FROM dbo.orders WHERE status <> 'shipped';\n```\n\nIt returns 4180. The table has 9000 rows and the shipped tile returns 3100, so 1720 orders are in neither tile.\n\nThe active order list runs:\n\n```sql\nSELECT * FROM dbo.orders WHERE id NOT IN (SELECT order_id FROM dbo.cancelled_orders);\n```\n\nIt returns zero rows, and only about forty orders have ever been cancelled.\n\nAnd `SELECT AVG(discount_pct) FROM dbo.orders` does not agree with `SELECT SUM(discount_pct) / COUNT(*) FROM dbo.orders`.\n\nExplain what is happening in each case and give me the corrected queries.", + "states": [ + "A comparison against a column with no value evaluates to unknown rather than true, so the filter excludes the row instead of keeping it.", + "The second query returns nothing because the subquery contains at least one row with no value, which makes the predicate unknown for every candidate row.", + "The average and the manual sum over row count differ because the aggregate skips rows with no value while the row count does not.", + "The engine accepts all three statements and returns a wrong answer rather than reporting anything to the client." + ], + "refutes": [ + "Azure SQL Database raises a warning to the client when an aggregate eliminates rows that hold no value, so this discrepancy would already have been visible in the query output.", + "NOT IN and NOT EXISTS return the same rows over a subquery whose column can hold no value, so that rewrite is a matter of style rather than correctness.", + "An inequality comparison already includes rows whose column holds no value, so the first tile is counting correctly and the missing 1720 rows must have been deleted or filtered somewhere else.", + "The rows are missing because of a collation difference, a data type mismatch or an implicit conversion in the WHERE clause." + ] + }, + "posture": ["read", "write"], + // Held at draft below a preview ceiling, on purpose. The reading of 2026-09-06 + // rested on three runs per arm, so it was re-measured on 2026-09-20 at ten before + // this field moved. Three statements at 10 of 10 in both arms, the fourth 8 of 10 + // against 7 of 10, every falsehood at 0 of 10 in both. The gate reads 9 of 10 + // with the skill against 10 of 10 without, so the model reaches this skill's own + // approach unaided. + // + // THE VERDICT IS NOW NEITHER, AND THAT IS WORSE FOR THIS SKILL, NOT BETTER. + // It read UNCLEAR until 2026-09-21 for one reason: the rule compared the gap + // as a proportion, and `0.8 - 0.7 > 0.1` is TRUE in binary floating point, so + // eight against seven fell out of "level" and into "undecided". The comparison + // is now made in whole counts, one tenth is one tenth, and the verdict + // re-derived from the same stored readings is NEITHER. UNCLEAR is a verdict a + // person may resolve with a decision file beside the report; NEITHER is not. + // Nothing about the skill changed. The arithmetic stopped hiding what the + // numbers already said. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "design-azure-sql-schema", + "t-sql-upserts-merge", + "t-sql-json-and-openjson", + "t-sql-regex-and-new-functions", + "t-sql-error-handling", + "t-sql-programmability-objects", + "post-migration-compatibility-level" + ], + "validation": { + "target": "both", + "setup": [ + "a database with one table carrying a nullable column and one carrying an identity column", + "a copy of the identity table with an enabled AFTER INSERT trigger" + ], + "assert": [ + "no predicate compares a nullable column or parameter with = or <> against a value that can be NULL, without IS NULL, IS NOT DISTINCT FROM or an explicit OR col IS NULL", + "no NOT IN names a subquery over a nullable column; NOT EXISTS is used instead", + "every average, ratio or percentage over a nullable column states whether NULL means zero or means excluded", + "no division intended to produce a fraction has integer operands on both sides", + "ISNULL is not substituted for COALESCE, or the reverse, without stating the intended return type", + "every character variable, parameter and cast carries an explicit length", + "the generated pagination uses ORDER BY with a unique tiebreaker plus OFFSET and FETCH, and does not combine TOP with OFFSET", + "every insert that needs its new key uses OUTPUT, with INTO when the target has an enabled trigger", + "no generated statement contains LIMIT, RETURNING, SERIAL, ILIKE, NOW(), true, false, ON CONFLICT or USE", + "string literals use single quotes and double quotes appear only around identifiers" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_null_cmp (id INT, status NVARCHAR(20) NULL);\nINSERT INTO dbo.probe_null_cmp VALUES (1, N'shipped'), (2, N'held'), (3, NULL);\nDECLARE @p NVARCHAR(20) = NULL;\nSELECT SUM(CASE WHEN status = @p THEN 1 ELSE 0 END) AS matched_by_equals,\n SUM(CASE WHEN status <> N'shipped' THEN 1 ELSE 0 END) AS not_shipped,\n SUM(CASE WHEN status IS NOT DISTINCT FROM @p THEN 1 ELSE 0 END) AS matched_null_safe\nFROM dbo.probe_null_cmp;", + "expect": { + "rows": [[0, 1, 1]] + }, + "why": "The claim the whole skill turns on, made executable in one row. Three rows exist, one of them NULL. Equality against a NULL parameter matches nothing, which is the documented ANSI_NULLS ON behaviour where every comparison against NULL is UNKNOWN and WHERE keeps only TRUE. The middle column is the one that reaches production: 'everything not shipped' is 1, not 2, because the NULL row is neither equal nor unequal. The third column is the fix, and IS NOT DISTINCT FROM is documented to return true or false and never unknown. Microsoft Learn: SET ANSI_NULLS truth table, and IS [NOT] DISTINCT FROM." + }, + { + "sql": "CREATE TABLE dbo.probe_assigned (customer_id INT NULL);\nINSERT INTO dbo.probe_assigned VALUES (1), (NULL);\nCREATE TABLE dbo.probe_customers (id INT NOT NULL);\nINSERT INTO dbo.probe_customers VALUES (1), (2), (3);\nSELECT (SELECT COUNT(*) FROM dbo.probe_customers c\n WHERE c.id NOT IN (SELECT customer_id FROM dbo.probe_assigned)) AS by_not_in,\n (SELECT COUNT(*) FROM dbo.probe_customers c\n WHERE NOT EXISTS (SELECT 1 FROM dbo.probe_assigned a WHERE a.customer_id = c.id)) AS by_not_exists;", + "expect": { + "rows": [[0, 2]] + }, + "why": "A single NULL in the subquery makes the NOT IN predicate UNKNOWN for every candidate row, so the answer is the empty set rather than a set short by one. Microsoft Learn's IN page carries this as a Caution: 'Any null values returned by subquery or expression that are compared to test_expression using IN or NOT IN return UNKNOWN'. The second column is the remedy the skill prescribes, proved to return the two customers a reader expects. Asserting both in one row is the point: the contrast is the correction, and a probe that measured only NOT IN would leave the fix unmeasured." + }, + { + "sql": "CREATE TABLE dbo.probe_readings (v INT NULL);\nINSERT INTO dbo.probe_readings VALUES (10), (NULL), (20);\nSELECT SUM(v) AS total, COUNT(*) AS all_rows, COUNT(v) AS non_null_rows, AVG(v) AS mean\nFROM dbo.probe_readings;", + "expect": { + "rows": [[30, 3, 2, 15]] + }, + "why": "The mean is not the total over the row count. AVG divides 30 by the two non-null rows and returns 15, where a reader reading COUNT(*) expects 10. Measured 2026-09-03 on EngineEdition 5, where every one of these four values came back with no message of any kind. COUNT(v) beside COUNT(*) in the same row is what makes the divisor visible. Sourcing note: the first draft of this cited a NULL semantics page that turned out to be Azure Databricks rather than this engine, so the claim now rests on the measurement and on the aggregate list in Microsoft Learn's SET ANSI_WARNINGS page." + }, + { + "sql": "SET ANSI_WARNINGS ON;\nCREATE TABLE dbo.probe_agg_silent (v INT NULL);\nINSERT INTO dbo.probe_agg_silent VALUES (10), (NULL), (20);\nSELECT (SELECT SUM(v) FROM dbo.probe_agg_silent) AS total,\n (SELECT COUNT(v) FROM dbo.probe_agg_silent) AS non_null_rows,\n (SELECT COUNT(*) FROM dbo.probe_agg_silent) AS all_rows,\n @@OPTIONS & 8 AS ansi_warnings_bit,\n (SELECT MAX(severity) FROM sys.messages WHERE message_id = 8153) AS msg_8153_severity;", + "expect": { + "rows": [[30, 2, 3, 8, 10]] + }, + "why": "This probe previously asserted warning 8153 and it FAILED against a real engine on 2026-09-03, EngineEdition 5, ProductVersion 12.0.2000.8, through sqlcmd -I -m-1, the same invocation that surfaces Msg 1945 for design-azure-sql-schema. The sum returned 30 and no message at all. So the probe now asserts what the engine does. The ansi_warnings_bit column is why the result can be trusted rather than dismissed as a session that had warnings off: @@OPTIONS & 8 is 8, so ANSI_WARNINGS genuinely is ON. The msg_8153_severity column is the sharpest part: message 8153 is defined in sys.messages at severity 10 and is simply never raised, which is the skill's own thesis about statements that succeed and return a wrong answer, stated by the engine's own catalog. Microsoft Learn disagrees, and the disagreement is the finding rather than something to smooth over: SET ANSI_WARNINGS says 'When set to ON, if null values appear in aggregate functions, such as SUM, AVG, MAX, MIN, STDEV, STDEVP, VAR, VARP, or COUNT, a warning message is generated', and ALTER DATABASE SET options repeats it. On this engine no warning is generated. COUNT(v) at 2 against COUNT(*) at 3 carries the divergence in the same row, so this probe stands alone if the one above it is read without it." + }, + { + "sql": "SELECT 7 / 2 AS integer_division, CAST(7.0 / 2 AS DECIMAL(4,1)) AS decimal_division;", + "expect": { + "rows": [[3, "3.5"]] + }, + "why": "Microsoft Learn's / (Division) page: the operator returns the data type of the argument with the higher precedence, and 'if an integer dividend is divided by an integer divisor, the result is an integer that has any fractional part of the result truncated'. No warning is raised, so a percentage or a rate built from two integer columns is simply wrong. The second column is cast to DECIMAL(4,1) rather than left as the raw literal type so the printed text is stable across builds rather than a probe about decimal scale." + }, + { + "sql": "DECLARE @short NVARCHAR(3) = NULL;\nSELECT ISNULL(@short, N'abcdef') AS isnull_result, COALESCE(@short, N'abcdef') AS coalesce_result;", + "expect": { + "rows": [["abc", "abcdef"]] + }, + "why": "ISNULL and COALESCE are treated as interchangeable and are not. Microsoft Learn's ISNULL page: the function returns the same type as check_expression, and 'replacement_value can be truncated if replacement_value is longer than check_expression'. COALESCE follows the CASE rules and returns the type with the highest precedence, so it returns all six characters. Swapping one for the other as a style edit changes the value returned, silently." + }, + { + "sql": "DECLARE @v NVARCHAR = N'abcdef';\nSELECT @v AS declared_without_length,\n LEN(CAST(REPLICATE(CAST(N'a' AS NVARCHAR(MAX)), 40) AS NVARCHAR)) AS cast_without_length;", + "expect": { + "rows": [["a", 30]] + }, + "why": "Microsoft Learn's nchar and nvarchar page: 'When n isn't specified in a data definition or variable declaration statement, the default length is 1. When n isn't specified with the CAST function, the default length is 30.' Both halves are asserted in one row because the surprise is that the same omitted length means two different things. The REPLICATE input is cast to NVARCHAR(MAX) first, without which REPLICATE returns nvarchar(4000) and a longer example would measure that limit instead of this one." + }, + { + "sql": "SET QUOTED_IDENTIFIER OFF;\nSELECT \"not a column\" AS literal_or_identifier;", + "expect": { + "rows": [["not a column"]] + }, + "why": "With QUOTED_IDENTIFIER OFF a double-quoted string is a literal, which Microsoft Learn's SET QUOTED_IDENTIFIER page states and demonstrates in its own example B. This is the half a reader meets on the container, whose own sqlcmd leaves the setting OFF while the ODBC and OLE DB drivers set it ON at connect. The pair of probes together is the claim: the same text, two different statements, decided by how the connection was made rather than by anything in the file." + }, + { + "sql": "SET QUOTED_IDENTIFIER ON;\nSELECT \"not a column\" AS literal_or_identifier;", + "expect": { + "error": 207 + }, + "why": "The other half. With the setting ON, which Learn calls the default and which every shipping driver sets at connect, the same double-quoted text is read as an object name and the statement fails. Microsoft Learn's 0 to 999 error table gives 207 at severity 16 as 'Invalid column name'. QUOTED_IDENTIFIER is stated in the batch because Learn documents that it takes effect at parse time and that an occurrence inside a top-level ad hoc batch changes parsing from that point on." + }, + { + "sql": "CREATE TABLE dbo.probe_out (id INT IDENTITY PRIMARY KEY, total DECIMAL(9,2));\nCREATE TABLE dbo.probe_out_log (id INT NULL);\nEXEC('CREATE TRIGGER dbo.probe_out_trg ON dbo.probe_out AFTER INSERT AS INSERT INTO dbo.probe_out_log (id) SELECT id FROM inserted;');\nINSERT INTO dbo.probe_out (total) OUTPUT INSERTED.id VALUES (19.99);", + "expect": { + "error": 334 + }, + "why": "The one restriction that separates OUTPUT from PostgreSQL's RETURNING, made executable. Microsoft Learn's OUTPUT clause page: if OUTPUT is specified without INTO, the target of the DML operation cannot have an enabled trigger for that action, and the 0 to 999 error table gives 334 at severity 15 as 'The target table of the DML statement cannot have any enabled triggers if the statement contains an OUTPUT clause without INTO clause'. CREATE TRIGGER must be first in its batch, so it is issued through EXEC rather than by relying on the runner honouring a GO." + }, + { + "sql": "CREATE TABLE dbo.probe_out2 (id INT IDENTITY PRIMARY KEY, total DECIMAL(9,2));\nCREATE TABLE dbo.probe_out2_log (id INT NULL);\nEXEC('CREATE TRIGGER dbo.probe_out2_trg ON dbo.probe_out2 AFTER INSERT AS INSERT INTO dbo.probe_out2_log (id) SELECT id FROM inserted;');\nDECLARE @new TABLE (id INT);\nINSERT INTO dbo.probe_out2 (total) OUTPUT INSERTED.id INTO @new VALUES (19.99);\nSELECT id FROM @new;", + "expect": { + "rows": [[1]] + }, + "why": "The remedy for the probe above, run end to end against the same fixture: the identical insert into the identical triggered table succeeds once OUTPUT captures into a table variable, and the new key is read back. Asserting the key rather than plain success proves the table variable was actually populated and not merely accepted by the parser." + }, + { + "sql": "SELECT ISNULL(N'a' || CAST(NULL AS NVARCHAR(10)), N'(null)') AS pipes_result,\n CONCAT(N'a', CAST(NULL AS NVARCHAR(10))) AS concat_result;", + "expect": { + "rows": [["(null)", "a"]] + }, + "why": "Two claims in one statement. That the ANSI double pipe operator exists here at all is the overcorrection half of this skill, and a failure on this probe means the engine predates its July 2025 general availability rather than that the claim is wrong. That it disagrees with CONCAT about NULL is the silent half: || propagates the null while CONCAT treats a null argument as an empty string, so the same name assembled two ways gives two answers." + } + , + { + "sql": "SELECT \"not a column\" AS x;", + "expect": { + "error": 207 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A double quoted string is an identifier under QUOTED_IDENTIFIER ON, which is the setting every driver sets, so the literal a developer typed is read as a column name. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_c_trg (id int, v int);\nGO\nCREATE TRIGGER probe_c_trg_ins ON dbo.probe_c_trg AFTER INSERT AS SELECT 1;\nGO\nINSERT INTO dbo.probe_c_trg (id, v) OUTPUT inserted.id VALUES (1, 1);", + "expect": { + "error": 334 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A bare OUTPUT is refused once the target carries an enabled trigger, so a pattern that worked in development breaks the day an audit trigger is added. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_c_p (id int NOT NULL PRIMARY KEY);\nCREATE TABLE dbo.probe_c_c (id int NOT NULL PRIMARY KEY, pid int NOT NULL REFERENCES dbo.probe_c_p(id));\nGO\nINSERT INTO dbo.probe_c_p (id) OUTPUT inserted.id INTO dbo.probe_c_c (id) VALUES (1);", + "expect": { + "error": 332 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. OUTPUT INTO refuses a target on either side of a foreign key, one of the three restrictions that make OUTPUT INTO narrower than it looks. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT NO_SUCH_FUNCTION_HERE(1) AS x;", + "expect": { + "error": 195 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. One of the three numbers a missing function can produce. The skill tells a reader to check the compatibility level before believing the function does not exist, and this is what an unrecognised name looks like. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT * FROM dbo.probe_c_no_such_object;", + "expect": { + "error": 208 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The second of the three, and the one OPENJSON produces below compatibility level 130, which is why the skill says read the level rather than the message. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_c_json (id int, j json);\nGO\nSELECT id FROM dbo.probe_c_json WHERE j = N'{}';", + "expect": { + "error": 402 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A json column compared with = is a type incompatibility, not a NULL problem, so it is a different failure from the ones this skill's first half is about. It goes red if the statement succeeds, or fails with any other number." + } + , + { + "sql": "CREATE TABLE dbo.probe_c_out_p (id int NOT NULL PRIMARY KEY);\nCREATE TABLE dbo.probe_c_out_c (id int NOT NULL, CONSTRAINT ck_probe_c CHECK (id > 0));\nGO\nINSERT INTO dbo.probe_c_out_p (id) OUTPUT inserted.id INTO dbo.probe_c_out_c (id) VALUES (1);", + "expect": { + "error": 333 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The second OUTPUT INTO restriction: the target can carry no enabled CHECK constraint. Together with the foreign key case this is why an audit table written by OUTPUT INTO has to be barer than a normal table. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_c_nulls (v int NULL);\nINSERT INTO dbo.probe_c_nulls VALUES (10), (NULL), (20);\nGO\nSET ANSI_WARNINGS ON;\nSELECT SUM(v) AS s, COUNT(*) AS c_all, COUNT(v) AS c_val, AVG(v) AS a FROM dbo.probe_c_nulls;", + "expect": { + "rows": [ + [ + 30, + 3, + 2, + 15 + ] + ] + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The aggregate claim, made executable: SUM skips the NULL and returns 30, COUNT(*) counts three rows while COUNT(v) counts two, and AVG divides by two rather than three. What this probe CANNOT assert is the other half of the claim, that message 8153 is never raised: the judge treats a severity 10 message as reportable and not as a failure, so a run where the engine started warning would still pass. That half stays a reading and not a check, and saying so is better than a probe that looks like it covers it. It goes red if any of the four values changes." + } + ] + }, + "triggering": { + "explicit": [ + "Use the t-sql-correctness skill before writing this query" + ], + "implicit": [ + "why does this WHERE clause skip rows that have a NULL in that column", + "why did my NOT IN query return no rows at all", + "why is this average different from the sum divided by the row count", + "why does this division return 0 instead of a fraction", + "write me a paginated query for Azure SQL Database", + "how do I get the id of the row I just inserted", + "port this PostgreSQL schema and these queries to Azure SQL Database", + "why does LIMIT 10 not work" + ], + "routing": [ + "translate this query for our SQL database", + "review this SQL before it goes in", + "should I use ISNULL or COALESCE here" + ], + "negative": [ + "write an upsert that inserts or updates a row", + "parse this JSON column with OPENJSON", + "pick the column types and collation for this new table", + "why is this query slow", + "write a PostgreSQL migration for my Postgres database" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson deleted file mode 120000 index 281b8aa..0000000 --- a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson +++ /dev/null @@ -1 +0,0 @@ -../../../skills/t-sql-json-and-openjson \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/SKILL.md b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/SKILL.md new file mode 100644 index 0000000..26ca865 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/SKILL.md @@ -0,0 +1,190 @@ +--- +name: t-sql-json-and-openjson +description: >- + Queries and stores JSON on Azure SQL Database using the native json type, a JSON index, and + OPENJSON with an explicit WITH schema, instead of the older nvarchar(max) plus JSON_VALUE + pattern the training data is full of. Use when asked to "store JSON in SQL", "query a JSON + column", "shred a JSON array into rows", "flatten this payload into a table", "index a JSON + property", "should this be nvarchar(max) or the json type", "parse the API response we saved", + or when JSON_VALUE, JSON_QUERY, JSON_MODIFY, ISJSON, OPENJSON, JSON_OBJECT or JSON_ARRAYAGG + appears in a query being written or reviewed; and when a JSON lookup returns NULL for a value + that is visibly present, or invalid JSON reached a column that nothing rejected. Where a + document column belongs in a table design is design-azure-sql-schema, and general T-SQL dialect + is t-sql-correctness. +--- + +# Query JSON on Azure SQL Database with the native type + +Two generations of JSON support exist here and a model has read far more of the first: a document +in `nvarchar(max)` read with functions. The second is a real `json` type with its own storage, +validation and index. Both work, and the differences are not only speed. Measured 2026-08-28 +against a live engine reporting `EngineEdition` 5 and Edition `SQL Azure`; the Learn pages for +`OPENJSON`, `JSON_VALUE` and `CREATE JSON INDEX` were re-read 2026-09-03. +Open [references/json-type-errors-and-timings.md](references/json-type-errors-and-timings.md) when +an error number you hit is not listed here, or before you change one that is. + +## What the json type actually changes + +| | `nvarchar(max)` | `json` | +|---|---|---| +| Invalid JSON on insert | accepted silently | rejected, `Msg 13609` | +| `CREATE JSON INDEX` | no, `Msg 13680` | yes, one per column | +| `JSON_VALUE(..., path RETURNING type)` | no, json input only | yes | +| `=`, `ORDER BY`, `GROUP BY`, `DISTINCT`, `LIKE` | yes | no: `Msg 402`, `13636`, `421`, `8116` | +| Text preserved byte for byte | yes | no, reformatted and duplicate keys dropped | + +**Validation.** `nvarchar(max)` takes `{not json at all` without a word, and the damage surfaces +months later as a `JSON_VALUE` returning `NULL` in a report. + +```sql +CREATE TABLE dbo.doc_nv (id int IDENTITY PRIMARY KEY CLUSTERED, payload nvarchar(max) NULL); +CREATE TABLE dbo.doc_json (id int IDENTITY PRIMARY KEY CLUSTERED, payload json NULL); +CREATE TABLE dbo.doc_chk (id int IDENTITY PRIMARY KEY CLUSTERED, payload nvarchar(max) NULL + CONSTRAINT ck_doc_chk_json CHECK (ISJSON(payload) = 1)); +INSERT INTO dbo.doc_nv (payload) VALUES (N'{not json at all'); -- no message of any kind +INSERT INTO dbo.doc_json (payload) VALUES (N'{not json at all'); -- Msg 13609 +INSERT INTO dbo.doc_chk (payload) VALUES (N'{not json at all'); -- Msg 547 +``` + +The constraint buys the refusal back for a column that must stay `nvarchar(max)`, and added later +it fails if a bad row is already there, which is the audit. `ISJSON` still returns 1 over a `json` +column; to demand a property rather than mere well-formedness Learn uses +`CHECK (JSON_PATH_EXISTS(payload, '$.basket') = 1)`. + +**Indexing.** The difference that cannot be worked around: + +```sql +CREATE TABLE dbo.orders_json (id int IDENTITY PRIMARY KEY CLUSTERED, payload json NOT NULL); +CREATE JSON INDEX jix_orders ON dbo.orders_json (payload) + FOR ('$.customer', '$.lines') WITH (OPTIMIZE_FOR_ARRAY_SEARCH = ON, FILLFACTOR = 80); +``` + +The same statement against `dbo.doc_nv` returns `Msg 13680`. Learn adds the fence: the table needs +a clustering key, one index per `json` column and 249 per table, the default path set is `$`, +listed paths may not overlap, builds are offline only. It serves `JSON_PATH_EXISTS`, +`JSON_CONTAINS` and `JSON_VALUE` compared with `=`, `RETURNING` cast included, but not `LIKE` or +`IS [NOT] NULL`. Learn's page omits Azure SQL Database from its applies-to banner +and calls the feature preview, yet it built and served queries on the engine measured here. + +One equality predicate on a nested property over 20,000 documents: 958 logical reads and 29 ms of +CPU on `nvarchar(max)`, 915 and 23 ms on `json` without a JSON index, **142 and 5 ms** with one. + +The alternative on either storage is the promoted column, a `PERSISTED` computed column over +`JSON_VALUE` with an ordinary index, which `design-azure-sql-schema` owns. Converting is close to +one way: `ALTER COLUMN payload json` fails with `Msg 13609` if any row is invalid, and the reverse +is refused in place with `Msg 257`. + +## OPENJSON, and why the WITH clause is not a style choice + +Without `WITH`, `OPENJSON` returns `key` in a BIN2 collation, `value` as `nvarchar(max)`, and +`type` numbered 0 null, 1 string, 2 number, 3 true/false, 4 array, 5 object, first level only. +With `WITH` it returns the columns asked for, typed, and those three are then unavailable. + +```sql +SELECT o.id, l.sku, l.qty, l.price, l.tags +FROM dbo.orders AS o +CROSS APPLY OPENJSON(o.payload, '$.lines') + WITH (sku nvarchar(10), qty int, price decimal(10,2), + tags nvarchar(max) '$.tags' AS JSON) AS l; +``` + +A path is optional where the column name equals the property name, as `sku` and `qty` show. +Shredding 40,000 array elements out of 20,000 `nvarchar(max)` documents cost **94 to 130 ms of CPU +with `WITH`, against 375 to 395 ms** for the default output plus a `JSON_VALUE` per field, across +three runs. The plans differ too, and reading them is `read-execution-plan`. Performance is the +smaller half: + +1. **A bad value fails loudly instead of arriving as text.** `WITH (qty int '$.qty')` over + `"qty": "abc"` raises `Msg 245`; the default output hands back `abc` as `nvarchar(max)`. +2. **A nested object or array needs `AS JSON`**, which requires `nvarchar(max)`; anything else is + `Msg 13618`. Without it the column is `NULL` in lax mode rather than the nested document. +3. **Paths and `key` are case-sensitive**, matched BIN2 and collation-unaware, so on a + case-insensitive database `WHERE [key] = N'orderid'` matched nothing against `orderId`. +4. **A missing property is `NULL` in lax mode and `Msg 13608` under `strict`**, and lax is the + default when no mode is written. Choose deliberately. + +## The 4000 character cliff, and the two ways past it + +`JSON_VALUE` without `RETURNING` is typed `nvarchar(4000)`, and Learn is explicit that above 4000 +characters it returns `NULL` in lax mode and an error in strict mode; measured, strict raised +`Msg 13625`, "would be truncated". That is data loss dressed as a missing key, and both escapes +fit in one statement: + +```sql +DECLARE @j json = JSON_OBJECT('big': REPLICATE(CAST(N'x' AS nvarchar(max)), 5000)); +SELECT JSON_VALUE(@j, '$.big') AS lax_returns_null, + LEN(JSON_VALUE(@j, '$.big' RETURNING nvarchar(max))) AS returning_len, + (SELECT LEN(v) FROM OPENJSON(@j) WITH (v nvarchar(max) '$.big')) AS openjson_len; +``` + +`RETURNING` is documented only for a `json` input, one more reason the storage choice is not +cosmetic; `OPENJSON ... WITH` works on both. It also takes the numeric, date and time types, so +`JSON_VALUE(payload, '$.customer.id' RETURNING int) = 16167` is both typed and indexable. + +## Compatibility level 130 is real, and almost never your problem + +`OPENJSON` is hidden below level 130 and returns `Msg 208`, `Invalid object name 'OPENJSON'`: +measured failing at 100 and 120, working at 130, while the `json` type and `CREATE JSON INDEX` +worked at 120. The requirement is real and nearly never the fault, because a database created on +Azure SQL Database today starts far above 130. + +```sql +SELECT DATABASEPROPERTYEX(DB_NAME(), 'CompatibilityLevel') AS level; +``` + +Read it that way rather than from `sys.databases`, observed stale in the same session as an +`ALTER DATABASE`. If the level cannot move, `post-migration-compatibility-level` owns raising it, +and Learn documents a second door: `ALLOW_BUILTIN_TVF_IN_ALL_COMPAT_LEVELS`, a database-scoped +configuration that exempts `OPENJSON` and the other built-in table-valued functions. + +## Check it worked + +Five traps from this page in one row: + +```sql +DECLARE @j json = N'{ "orderId" : 7, "customer" : {"id":"C1"}, "dup": 1, "dup": 2 }'; +SELECT CAST(@j AS nvarchar(max)) AS normalized, + JSON_VALUE(@j, '$.orderid') AS wrong_case, + JSON_VALUE(@j, '$.orderId') AS right_case, + (SELECT c FROM OPENJSON(@j) WITH (c nvarchar(max) '$.customer')) AS without_as_json, + (SELECT c FROM OPENJSON(@j) WITH (c nvarchar(max) '$.customer' AS JSON)) AS with_as_json; +``` + +```text +normalized {"orderId":7,"customer":{"id":"C1"},"dup":1} +wrong_case NULL +right_case 7 +without_as_json NULL +with_as_json {"id":"C1"} +``` + +Whitespace and a second `dup` in `normalized` mean the column is `nvarchar(max)`, not `json`; a +`NULL` in `right_case` means the property is spelled some other way. In a pipeline pass both error +flags, because `-b` alone sets a non-zero exit only above severity 10 and without `-m-1` a +severity 10 message prints with no `Msg` number to grep for: + +```bash +sqlcmd -S ,1433 -d -b -m-1 -i check-json.sql +``` + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value, so on that build there is no number to grep for whatever `-m` says. The +numbers in this skill that matter most, `13609`, `13618`, `13608` and `13625`, are all severity 16 +and print on either build; it is the severity 10 case this flag is for. +`build-app-on-azure-sql` tells the two builds apart in one table. + +## Do not + +- Do not default to `nvarchar(max)` because it is familiar: it accepts invalid JSON, cannot carry + a JSON index, and `RETURNING` is closed to it. +- Do not call `nvarchar(max)` plus functions deprecated. It is not, and on scalar reads it is the + cheaper plan: four properties from each of 20,000 rows cost **81 ms** of CPU with `JSON_VALUE` + against **735 ms** for `OPENJSON ... WITH`. +- Do not reach for `OPENJSON` to read one scalar property, and do not lower-case a path or a `key` + comparison, which quietly matches nothing. +- Do not assume a `json` column hands back the text that went in. It is reparsed, whitespace goes + and a duplicate key is dropped, so if a signature covers the exact bytes keep the raw text too. +- Do not review only the query. An unvalidated `nvarchar(max)`, a missing `WITH` or `AS JSON`, and + a lower-cased path are all invisible in the result set. diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/references/json-type-errors-and-timings.md b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/references/json-type-errors-and-timings.md new file mode 100644 index 0000000..5e78d30 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/references/json-type-errors-and-timings.md @@ -0,0 +1,304 @@ +# The json type, JSON indexes and OPENJSON: errors, limits and timings + +## Contents + +- [How this was measured](#how-this-was-measured) +- [Validation, or the absence of it](#validation-or-the-absence-of-it) +- [Round trip and normalization](#round-trip-and-normalization) +- [Storage size](#storage-size) +- [What a json column cannot do](#what-a-json-column-cannot-do) +- [The JSON index](#the-json-index) +- [Compatibility levels](#compatibility-levels) +- [The 4000 character cliff](#the-4000-character-cliff) +- [OPENJSON default output versus an explicit schema](#openjson-default-output-versus-an-explicit-schema) +- [Timing table](#timing-table) +- [Converting an existing column](#converting-an-existing-column) +- [What is documented rather than measured](#what-is-documented-rather-than-measured) + +## How this was measured + +One engine reporting: + +```text +SERVERPROPERTY('EngineEdition') = 5 +SERVERPROPERTY('Edition') = SQL Azure +``` + +Date of the run: 2026-08-28. Two 20,000 row tables holding identical documents, one column typed +`JSON` and one `NVARCHAR(MAX)`, each document an order with a nested customer object and a two +element line array. Timings are `SET STATISTICS TIME` CPU, reads are `SET STATISTICS IO` logical +reads, and every timing below was taken at least twice. + +## Validation, or the absence of it + +```sql +INSERT INTO dbo.doc_nv (payload) VALUES (N'{not json at all'); -- accepted +INSERT INTO dbo.doc_json (payload) VALUES (N'{not json at all'); +``` + +```text +Msg 13609 JSON text is not properly formatted. Unexpected character 'n' is found at position 1. +``` + +The `nvarchar(max)` insert produced no message of any kind. The substitute, on a table that must +keep `nvarchar(max)`: + +```sql +CREATE TABLE dbo.nvchk ( + id INT IDENTITY PRIMARY KEY, + payload NVARCHAR(MAX) NULL CONSTRAINT ck_json CHECK (ISJSON(payload) = 1) +); +INSERT INTO dbo.nvchk (payload) VALUES (N'not json'); +``` + +```text +Msg 547 The INSERT statement conflicted with the CHECK constraint "ck_json". +``` + +`ISJSON` over a native `json` column returns 1, so the function is not made redundant by the type. + +## Round trip and normalization + +Input, into both columns: + +```text +{ "b" : 1, "a" : 2, "dup": 1, "dup": 2 } +``` + +Read back: + +```text +json -> {"b":1,"a":2,"dup":1} +nvarc -> { "b" : 1, "a" : 2, "dup": 1, "dup": 2 } +``` + +Whitespace removed, key order preserved in this case, and the second `"dup"` dropped. `JSON_VALUE` +over the duplicate key returned `1` on both storages, so the two agree on which duplicate wins, +but only the `json` column discards the loser permanently. + +## Storage size + +`DATALENGTH` on the same content: + +| Content | `nvarchar(max)` | `json` | +|---|---|---| +| `{"b":1,"a":2,"dup":1}` | 42 | 79 | +| A realistic order document | 506 | 430 | +| An array of 51 small objects | 3866 | 2205 | + +At table scale, 20,000 rows: `orders_nv` used 7664 KB, `orders_json` used 7320 KB. The binary +format has a fixed floor that a tiny document cannot amortize, and pays back on documents with +repeated keys. + +## What a json column cannot do + +Each statement was run in its own batch, because these are compile time failures that abort the +batch rather than errors a `TRY` block can catch. + +```text +SELECT COUNT(*) FROM dbo.doc_json WHERE payload = N'{"b":1}'; +Msg 402 The data types json and nvarchar are incompatible in the equal to operator. + +SELECT COUNT(*) FROM dbo.doc_json GROUP BY payload; +Msg 13636 The JSON data type cannot be compared or sorted, except when using the IS NULL operator. + +SELECT TOP 1 id FROM dbo.doc_json ORDER BY payload; +Msg 13636 (same) + +SELECT TOP 1 id FROM dbo.doc_json WHERE payload LIKE '%a%'; +Msg 8116 Argument data type json is invalid for argument 1 of like function. + +SELECT DISTINCT payload FROM dbo.doc_json; +Msg 421 The json data type cannot be selected as DISTINCT because it is not comparable. + +SELECT ... FROM dbo.doc_json a JOIN dbo.doc_json b ON a.payload = b.payload; +Msg 13636 (same) +``` + +A `json` column **is** accepted as an `INCLUDE` column and in the `WHERE` clause of a filtered +index. Both were created cleanly. + +## The JSON index + +```sql +CREATE JSON INDEX jix_orders ON dbo.orders_json (payload); -- succeeds +CREATE JSON INDEX jix_doc2 ON dbo.doc_json (payload) FOR ('$.a'); +``` + +```text +Msg 13681 A JSON index 'jix_doc' already exists on column 'payload' on table 'doc_json', + and multiple JSON indexes per column are not allowed. +``` + +On the `nvarchar(max)` column: + +```text +Msg 13680 Column 'payload' on table 'dbo.orders_nv' is not of JSON data type, + which is required to create a JSON index on it. +``` + +One predicate, `WHERE JSON_VALUE(payload, '$.customer.id') = 'C500'`, 20,000 rows: + +| Storage | logical reads | CPU | +|---|---|---| +| `nvarchar(max)` | 958 on the base table | 29 ms | +| `json`, no JSON index | 915 on the base table | 23 ms | +| `json` with a JSON index | 142, on an object named `json_index_` | 5 ms | + +The index is maintained on write. After `UPDATE ... SET payload = N'{"t":"bronze"}'` on a row that +had been `gold`, a filter for `gold` returned 0 and a filter for `bronze` returned 1. + +## Compatibility levels + +A database set to each level in turn: + +| Level | `json` type | `CREATE JSON INDEX` | `JSON_VALUE` | `JSON_OBJECT`, `JSON_ARRAYAGG` | `OPENJSON` | +|---|---|---|---|---|---| +| 100 | not retested | not retested | not retested | not retested | **`Msg 208`** | +| 120 | works | works | works | works | **`Msg 208`** | +| 130 | works | works | works | works | works | +| 170 | works | works | works | works | works | + +```text +Msg 208, Level 16, State 1 +Invalid object name 'OPENJSON'. +``` + +Read the level with `DATABASEPROPERTYEX(DB_NAME(), 'CompatibilityLevel')` rather than +`sys.databases`, which was observed returning a stale value immediately after an `ALTER DATABASE` +in the same session. + +## The 4000 character cliff + +A document holding one 5000 character string property: + +```text +doc length = 5010 +JSON_VALUE lax -> NULL (silently) +JSON_VALUE over the json column type -> NULL (silently) +OPENJSON WITH (v NVARCHAR(MAX) ...) -> len 5000 +``` + +Under a `strict` path: + +```text +Msg 13625 String value in the specified JSON path would be truncated. +``` + +Both storages behave identically, so this is a `JSON_VALUE` property, not an `nvarchar` property. +The return type is capped, and lax mode turns the overflow into `NULL`. + +## OPENJSON default output versus an explicit schema + +Default output over `{"orderId":1,"customer":{...},"lines":[...],"placedAt":"..."}`: + +```text +orderId | 1 | type=2 +customer | {"id":"C1","tier":"silver"} | type=5 +lines | [{"sku":"A-100",... | type=4 +placedAt | 2026-01-02T10:00:00Z | type=1 +``` + +Described result set: + +```text +key nvarchar(8000) collation = Latin1_General_BIN2 +value nvarchar(max) collation = SQL_Latin1_General_CP1_CI_AS +type tinyint +``` + +The `key` collation is the trap. On a case-insensitive database: + +```text +WHERE [key] = N'orderId' -> 1 row +WHERE [key] = N'orderid' -> 0 rows +``` + +Paths behave the same way: + +```text +JSON_VALUE('{"orderId":1}', '$.orderid') -> NULL +JSON_VALUE('{"orderId":1}', '$.orderId') -> 1 +``` + +Behaviour of the `WITH` clause: + +```text +WITH (qty INT '$.qty') over "qty":"abc" + Msg 245 Conversion failed when converting the nvarchar value 'abc' to data type int. + +WITH (qty INT '$.qty') over a document with no qty -> NULL +WITH (qty INT 'strict $.qty') over a document with no qty + Msg 13608 Property cannot be found on the specified JSON path. + +WITH (lines NVARCHAR(100) '$.lines' AS JSON) + Msg 13618 AS JSON option can be specified only for column of nvarchar(max) type in WITH clause. + +WITH (lines NVARCHAR(MAX) '$.lines' AS JSON) -> [1,2] +WITH (c NVARCHAR(MAX) '$.c') over "c":{"x":1} -> NULL +``` + +That last line is the quiet one: a nested object requested without `AS JSON` comes back as `NULL` +rather than as its text, and lax mode raises nothing. + +## Timing table + +20,000 documents, 40,000 array elements. CPU milliseconds, best of at least two runs. + +| Work | Storage | Shape | CPU | +|---|---|---|---| +| Shred the `lines` array | `nvarchar(max)` | `OPENJSON ... WITH` | 94 to 130 | +| Shred the `lines` array | `nvarchar(max)` | default output plus `JSON_VALUE` | 375 to 395 | +| Shred the `lines` array | `json` | `OPENJSON ... WITH` | 790 to 1105 | +| Shred the `lines` array | `json` | default output plus `JSON_VALUE` | 927 to 1259 | +| Four scalar properties, every row | `json` | `JSON_VALUE` four times | 81 to 120 | +| Four scalar properties, every row | `json` | `OPENJSON ... WITH` | 592 to 735 | +| Four scalar properties, every row | `nvarchar(max)` | `JSON_VALUE` four times | 114 to 157 | + +Two conclusions, and the second is the one that is easy to get backwards: + +- For array shredding on `nvarchar(max)` storage, which is what a model writes, the explicit schema + is about four times cheaper than the default output plus `JSON_VALUE`. +- For scalar property reads there is no array to shred, and `OPENJSON` is the **more** expensive + plan on both storages. `JSON_VALUE` is the right tool there, and it is the only one a JSON index + can serve. + +Array shredding on the `json` column measured slower than on `nvarchar(max)` in every run. That is +reproducible here but not explained, so it is recorded rather than turned into advice. + +## Converting an existing column + +```sql +ALTER TABLE dbo.mig ALTER COLUMN payload JSON NULL; +``` + +With one invalid row present: + +```text +Msg 13609 JSON text is not properly formatted. Unexpected character 'n' is found at position 0. +The statement has been terminated. +``` + +After `DELETE FROM dbo.mig WHERE ISJSON(payload) = 0`, the same statement succeeded and +`sys.types` reported `json`. Reversing it: + +```text +ALTER TABLE dbo.mig ALTER COLUMN payload NVARCHAR(MAX) NULL; +Msg 257 Implicit conversion from data type json to nvarchar(max) is not allowed. + Use the CONVERT function to run this query. +``` + +So the conversion out needs a new column and an explicit `CAST`, not an in place `ALTER`. Plan the +move as a one way door and keep the raw text separately if anything depends on it. + +## What is documented rather than measured + +Read on 2026-08-28 for Azure SQL Database, not reproduced here. + +- The `json` type is generally available, is available under all database compatibility levels, + and stores up to 2 GB. +- A `json` column is permitted as an included column and in the `WHERE` clause of a filtered index. + Both were confirmed here; the size table on the documentation page was not copied. +- `design-azure-sql-schema` measured the key restrictions on the same engine: `Msg 1978` for a + `json` column used as an index key, and `Msg 402` for an equality comparison against `nvarchar`. + This page confirms both rather than re-deriving them. diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/skill.spec.jsonc new file mode 100644 index 0000000..a4bec0b --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-json-and-openjson/skill.spec.jsonc @@ -0,0 +1,336 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "t-sql-json-and-openjson", + "domain": "tsql-correctness-and-patterns", + "value": ["not-in-training-data", "corrects-a-wrong-assumption"], + "correction": "Asked to store or query JSON on Azure SQL Database, an agent reaches for nvarchar(max) with JSON_VALUE, because that is the pattern almost all of its training data shows, and it treats the native json type as either absent or as a cosmetic rename. Measured on 2026-08-28 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure', the type changes four things that matter and the wrong choice is not merely slower. An nvarchar(max) column accepted the text '{not json at all' with no message at all, while the json column rejected it with Msg 13609, so the older pattern turns a bad write into a JSON_VALUE that returns NULL in a report months later, unless someone remembered CHECK (ISJSON(payload) = 1), which does work and returns Msg 547. CREATE JSON INDEX is impossible on nvarchar(max), Msg 13680, and on the native type it took one equality predicate over 20,000 documents from 915 logical reads and 23 ms of CPU to 142 reads and 5 ms. The json column stores a parsed document rather than the text it was given, so a round trip is not byte for byte and a duplicate key is silently discarded, and the conversion back out is refused in place with Msg 257. The second half of the correction is OPENJSON. An agent that does use OPENJSON usually omits the WITH clause and then calls JSON_VALUE on the default value column, which measured about four times the CPU for array shredding on nvarchar storage, 94 to 130 ms against 375 to 395 ms; but the correctness gap is larger than the performance one, because JSON_VALUE returns NULL rather than an error or a prefix for any value over 4000 characters in lax mode, while OPENJSON WITH an NVARCHAR(MAX) column returned all 5000, and because the default output's key column carries the Latin1_General_BIN2 collation so a lower-cased key comparison matched zero rows on a case-insensitive database. The reverse overcorrection has to be stated too: OPENJSON is not the answer to everything, and reading four scalar properties from every row cost 81 ms with JSON_VALUE against 735 ms with OPENJSON WITH over the same rows.", + "value_probe": { + "gate": "The answer changes how the document is stored so a bad write is refused at write time and the property filter can be served without reading every row, and it retrieves the long value by a path that is not capped at 4000 characters. Keeping the column as it is, with no validation and no index strategy, has not reached it.", + "task": "We save vendor webhook payloads on Azure SQL Database in dbo.webhook_events (id bigint, received_at datetime2, payload nvarchar(max)). Two support problems and one worry.\n\nA report pulls the customer note out with `JSON_VALUE(payload, '$.customer.notes')` and gets nothing back for about three percent of rows. When I open those rows the note text is plainly there in the document, it is just long, several thousand characters. No error is raised, the column is simply empty in the report.\n\nSeparately, we found rows whose payload is not a document at all, a truncated string beginning with `{` that was never finished. Nothing complained when it was written and we have no idea how long they have been there.\n\nAnd we are about to add a filter on `$.status` over roughly twenty million rows, which I am nervous about.\n\nThis is a current Azure SQL Database, created this year. Tell me how to fix the storage and the queries, with the SQL you would actually run.", + "states": [ + "The note is missing because the extraction path has a value length ceiling and returns nothing rather than an error or a shortened value when a value exceeds it.", + "A column declared as nvarchar accepts text that is not a valid document, so the invalid rows were never going to be refused without an explicit validation on the column.", + "There is a storage choice on a current Azure SQL Database that validates the document on write, and it is not nvarchar(max).", + "The property filter over twenty million rows needs an index built for it, and what is available depends on how the column is typed." + ], + "refutes": [ + "The extraction function returns the first part of a value that is too long, or raises an error on it, so the empty results must mean the property is genuinely absent from those documents.", + "nvarchar(max) is the only way to store a document on Azure SQL Database, and any dedicated document type is either unavailable there or just an alias for nvarchar with no behaviour of its own.", + "A document column cannot be indexed on Azure SQL Database, so the only way to make the property filter fast is a persisted computed column with an ordinary index on it.", + "Storing the document preserves the text exactly as it was submitted, so what comes back out is byte for byte what the vendor sent." + ] + }, + "posture": ["read", "write"], + "maturity": "ga", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "t-sql-correctness", + "t-sql-upserts-merge", + "design-azure-sql-schema", + "post-migration-compatibility-level", + "dab-rest-and-graphql", + "diagnose-slow-query" + ], + "validation": { + "target": "both", + "setup": [ + "a database at compatibility level 130 or above", + "a table holding the same documents twice, once in a json column and once in nvarchar(max)", + "documents carrying a nested object, an array of objects, and one string property longer than 4000 characters", + "enough rows that a scan is measurably different from an index read" + ], + "assert": [ + "a new document column is declared json, or the answer states why nvarchar(max) was chosen and adds a CHECK (ISJSON(...) = 1) constraint", + "no generated statement compares, sorts, groups, applies DISTINCT to, or uses LIKE against a json column", + "every OPENJSON that turns an array into rows carries a WITH clause naming typed columns", + "no property that can exceed 4000 characters is read with JSON_VALUE", + "a nested object or array pulled through OPENJSON WITH uses NVARCHAR(MAX) with AS JSON", + "every JSON path matches the case of the property in the sample document", + "a single scalar property read is JSON_VALUE rather than an OPENJSON construction", + "a property that every query filters on is either promoted to an indexed computed column or covered by a JSON index on a native json column", + "no answer describes the nvarchar plus functions pattern as removed or deprecated" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_doc_json (id INT IDENTITY PRIMARY KEY CLUSTERED, payload JSON NULL);\nINSERT INTO dbo.probe_doc_json (payload) VALUES (N'{not json at all');", + "expect": { + "error": 13609 + }, + "why": "The correction the skill turns on, made executable. A json column parses on write and refuses text that is not a document. Measured 2026-08-28, message \"JSON text is not properly formatted. Unexpected character 'n' is found at position 1.\"" + }, + { + "sql": "CREATE TABLE dbo.probe_doc_nv (id INT IDENTITY PRIMARY KEY CLUSTERED, payload NVARCHAR(MAX) NULL);\nINSERT INTO dbo.probe_doc_nv (payload) VALUES (N'{not json at all');\nSELECT COUNT(*) AS rows_accepted FROM dbo.probe_doc_nv;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The half that makes the probe above mean something. The same text into nvarchar(max) is stored with no message of any kind, so the row lands and the damage surfaces later as a JSON_VALUE returning NULL. Measured 2026-08-28. Asserting the refusal without asserting the silence leaves the reader no reason to switch." + }, + { + "sql": "CREATE TABLE dbo.probe_doc_chk (id INT IDENTITY PRIMARY KEY CLUSTERED, payload NVARCHAR(MAX) NULL\n CONSTRAINT ck_probe_doc_chk CHECK (ISJSON(payload) = 1));\nINSERT INTO dbo.probe_doc_chk (payload) VALUES (N'{not json at all');", + "expect": { + "error": 547 + }, + "why": "The escape for a column that must stay nvarchar(max): CHECK (ISJSON(payload) = 1) buys the refusal back, as a constraint violation rather than a parse failure. Measured 2026-08-28." + }, + { + "sql": "CREATE TABLE dbo.probe_sort (id INT IDENTITY PRIMARY KEY CLUSTERED, payload JSON NOT NULL);\nINSERT INTO dbo.probe_sort (payload) VALUES (N'{\"a\":1}');\nSELECT TOP 1 id FROM dbo.probe_sort ORDER BY payload;", + "expect": { + "error": 13636 + }, + "why": "The json type is not comparable, so ORDER BY, GROUP BY and a join on the column all fail the same way. Measured 2026-08-28, message \"The JSON data type cannot be compared or sorted, except when using the IS NULL operator.\" design-azure-sql-schema already measured Msg 402 for equality against nvarchar; this is the sort and group half, not a restatement of it." + }, + { + "sql": "CREATE TABLE dbo.probe_like (id INT IDENTITY PRIMARY KEY CLUSTERED, payload JSON NOT NULL);\nINSERT INTO dbo.probe_like (payload) VALUES (N'{\"a\":1}');\nSELECT TOP 1 id FROM dbo.probe_like WHERE payload LIKE '%a%';", + "expect": { + "error": 8116 + }, + "why": "LIKE over a json column is refused rather than falling back to the text, which is the trap for anyone porting a query that grepped an nvarchar(max) document. Measured 2026-08-28, \"Argument data type json is invalid for argument 1 of like function.\"" + }, + { + "sql": "CREATE TABLE dbo.probe_ix_nv (id INT IDENTITY PRIMARY KEY CLUSTERED, payload NVARCHAR(MAX) NOT NULL);\nCREATE JSON INDEX jix_probe_nv ON dbo.probe_ix_nv (payload);", + "expect": { + "error": 13680 + }, + "why": "The difference the skill says cannot be worked around. A JSON index requires the native type, so the storage choice closes this door permanently. Measured 2026-08-28, \"Column 'payload' ... is not of JSON data type, which is required to create a JSON index on it.\"" + }, + { + "sql": "CREATE TABLE dbo.probe_ix_json (id INT IDENTITY PRIMARY KEY CLUSTERED, payload JSON NOT NULL);\nCREATE JSON INDEX jix_probe_json ON dbo.probe_ix_json (payload)\n FOR ('$.customer', '$.lines') WITH (OPTIMIZE_FOR_ARRAY_SEARCH = ON, FILLFACTOR = 80);", + "expect": { + "succeeds": true + }, + "why": "The statement the body tells the reader to run, with the two options Learn documents on the CREATE JSON INDEX page, asserted end to end. This one settles a live disagreement rather than padding a count: that Learn page omits Azure SQL Database from its applies-to banner and calls the feature preview, yet the statement built on the engine measured 2026-08-28, which reported EngineEdition 5 and Edition 'SQL Azure'. If it ever stops building, the body's central recommendation is wrong and this probe is how we find out." + }, + { + "sql": "SELECT qty FROM OPENJSON(N'[{\"qty\":\"abc\"}]') WITH (qty INT '$.qty');", + "expect": { + "error": 245 + }, + "why": "The correctness argument for the WITH clause, which the body ranks above the speed argument. A typed column refuses a value that is not that type; the default output would hand back 'abc' as nvarchar(max) and let it travel. Measured 2026-08-28." + }, + { + "sql": "SELECT qty FROM OPENJSON(N'[{\"sku\":\"A-100\"}]') WITH (qty INT 'strict $.qty');", + "expect": { + "error": 13608 + }, + "why": "The lax and strict pairing. Lax is the default when no mode is written and returns NULL for a property that is absent; strict raises. Measured 2026-08-28, \"Property cannot be found on the specified JSON path.\" The same document read without the strict keyword returns a NULL row instead." + }, + { + "sql": "SELECT lines FROM OPENJSON(N'[{\"lines\":[1,2]}]') WITH (lines NVARCHAR(100) '$.lines' AS JSON);", + "expect": { + "error": 13618 + }, + "why": "AS JSON is the only way to pull a nested object or array through the WITH clause, and it accepts one type. Measured 2026-08-28 and stated verbatim on Microsoft Learn's error list: \"AS JSON option can be specified only for column of nvarchar(max) type in WITH clause.\" Written as NVARCHAR(100) here because that is the mistake, not an invented one." + }, + { + "sql": "DECLARE @j JSON = JSON_OBJECT('big': REPLICATE(CAST(N'x' AS NVARCHAR(MAX)), 5000));\nSELECT ISNULL(LEN(JSON_VALUE(@j, '$.big')), -1) AS lax_len,\n LEN(JSON_VALUE(@j, '$.big' RETURNING NVARCHAR(MAX))) AS returning_len,\n (SELECT LEN(v) FROM OPENJSON(@j) WITH (v NVARCHAR(MAX) '$.big')) AS openjson_len;", + "expect": { + "rows": [ + [ + -1, + 5000, + 5000 + ] + ] + }, + "why": "The 4000 character cliff and both escapes in one statement. JSON_VALUE without RETURNING is typed nvarchar(4000) and lax mode turns the overflow into NULL, reported here as -1 so the row is unambiguous; RETURNING nvarchar(max) and OPENJSON WITH both return all 5000. Measured 2026-08-28, and Learn states the lax NULL and the strict error on the JSON_VALUE page. The CAST inside REPLICATE is load-bearing: without it REPLICATE returns nvarchar(4000) and the fixture silently becomes NULL." + }, + { + "sql": "DECLARE @j JSON = JSON_OBJECT('big': REPLICATE(CAST(N'x' AS NVARCHAR(MAX)), 5000));\nSELECT JSON_VALUE(@j, 'strict $.big');", + "expect": { + "error": 13625 + }, + "why": "The other side of the cliff, and the reason the body calls the lax behaviour data loss dressed as a missing key. Under a strict path the same read raises rather than returning NULL. Measured 2026-08-28, \"String value in the specified JSON path would be truncated.\"" + }, + { + "sql": "DECLARE @j JSON = N'{\"orderId\":7}';\nSELECT CASE WHEN JSON_VALUE(@j, '$.orderid') IS NULL THEN 1 ELSE 0 END AS wrong_case_is_null,\n JSON_VALUE(@j, '$.orderId' RETURNING INT) AS right_case;", + "expect": { + "rows": [ + [ + 1, + 7 + ] + ] + }, + "why": "JSON paths are case-sensitive regardless of the database collation, so a lower-cased path matches nothing and returns NULL rather than erroring. Measured 2026-08-28 on a case-insensitive database, where the OPENJSON key column carries Latin1_General_BIN2 and behaves the same way. This is the failure the description lists as 'JSON_VALUE returns NULL but the property is right there'." + }, + { + "sql": "SELECT COUNT(*) AS openjson_rows FROM OPENJSON(N'{\"a\":1,\"b\":2}');", + "expect": { + "rows": [ + [ + 2 + ] + ] + }, + "why": "The compatibility level 130 requirement is real, and this asserts the half that matters to a reader: OPENJSON resolves on a database created normally, with nobody having run ALTER DATABASE SET COMPATIBILITY_LEVEL first. Measured 2026-08-28 failing with Msg 208 at levels 100 and 120 and working from 130 up, which Learn states on the OPENJSON page. Deliberately does not assert the level number, because a freshly created database's default moves over time and a pinned value would be a single sample." + }, + { + "sql": "DECLARE @j JSON = N'{ \"orderId\" : 7, \"dup\": 1, \"dup\": 2 }';\nSELECT CAST(@j AS NVARCHAR(MAX)) AS normalized;", + "expect": { + "rows": [ + [ + "{\"orderId\":7,\"dup\":1}" + ] + ] + }, + "why": "A json column stores a parsed document, not the bytes it was given: whitespace goes and the second duplicate key is discarded. Measured 2026-08-28. This is the check the body's 'Check it worked' section reads first, because it is how a reader tells which type they actually have, and the reason the body says to keep the raw text separately if a signature covers the exact bytes." + } + , + { + "sql": "CREATE TABLE dbo.probe_j_json (payload json);\nGO\nINSERT INTO dbo.probe_j_json (payload) VALUES (N'{not json at all');", + "expect": { + "error": 13609 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The json type rejects invalid text on insert. The whole first table in this skill turns on this being an engine rule rather than a habit, because nvarchar(max) accepts the same string silently. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_chk (payload nvarchar(max) CONSTRAINT ck_probe_j CHECK (ISJSON(payload) = 1));\nGO\nINSERT INTO dbo.probe_j_chk (payload) VALUES (N'{not json at all');", + "expect": { + "error": 547 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The nvarchar(max) alternative needs a CHECK constraint to get the same refusal, and it arrives as a constraint violation rather than a JSON error, which is the difference in diagnosis this skill draws. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_nv (id int NOT NULL PRIMARY KEY, payload nvarchar(max));\nGO\nCREATE JSON INDEX probe_j_nv_ix ON dbo.probe_j_nv (payload);", + "expect": { + "error": 13680 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A JSON index needs the json type, so an nvarchar(max) column cannot carry one. This is the cost of the familiar choice, stated as a refusal. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_idx (id int NOT NULL PRIMARY KEY, payload json);\nGO\nCREATE JSON INDEX probe_j_idx_ix ON dbo.probe_j_idx (payload);", + "expect": { + "succeeds": true + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The other half of the pair above, so the refusal on nvarchar(max) is read as a type rule and not as JSON indexes being unavailable on this engine. It asserts the build itself rather than a row count in sys.indexes, which carries two rows for one JSON index on this build, a shape nothing here claims. It goes red if CREATE JSON INDEX ever raises." + }, + { + "sql": "CREATE TABLE dbo.probe_j_eq (id int, payload json);\nGO\nSELECT id FROM dbo.probe_j_eq WHERE payload = N'{}';", + "expect": { + "error": 402 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A json column cannot be compared with =, which is the first row of the comparison list and the reason a signature or a dedupe key cannot live on the column itself. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_ord (id int, payload json);\nGO\nSELECT id FROM dbo.probe_j_ord ORDER BY payload;", + "expect": { + "error": 13636 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Neither ORDER BY nor GROUP BY works on a json column. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_dis (id int, payload json);\nGO\nSELECT DISTINCT payload FROM dbo.probe_j_dis;", + "expect": { + "error": 421 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. SELECT DISTINCT over a json column is refused for the same reason. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_like (id int, payload json);\nGO\nSELECT id FROM dbo.probe_j_like WHERE payload LIKE N'%a%';", + "expect": { + "error": 8116 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. LIKE is refused too, which closes the last habit a developer reaches for when the other three fail. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_rev (id int, payload json);\nGO\nALTER TABLE dbo.probe_j_rev ALTER COLUMN payload nvarchar(max);", + "expect": { + "error": 257 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The migration runs one way. Coming back out of json needs an explicit CONVERT and a rebuild, which is what makes the choice worth making deliberately. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "CREATE TABLE dbo.probe_j_fwd (id int, payload nvarchar(max));\nINSERT INTO dbo.probe_j_fwd VALUES (1, N'{not json');\nGO\nALTER TABLE dbo.probe_j_fwd ALTER COLUMN payload json;", + "expect": { + "error": 13609 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. ALTER COLUMN to json validates every existing row, so a table that accepted junk for years fails the migration on the first bad row rather than converting it. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT qty FROM OPENJSON(N'{\"qty\":\"abc\"}') WITH (qty int '$.qty');", + "expect": { + "error": 245 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A typed WITH column fails loudly on a bad value where the default output hands the same text back as nvarchar(max), which is the argument for WITH. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT c FROM OPENJSON(N'{\"c\":{\"id\":1}}') WITH (c nvarchar(50) '$.c' AS JSON);", + "expect": { + "error": 13618 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. AS JSON requires nvarchar(max) and nothing narrower, so the natural first attempt at reading a nested document is refused rather than silently truncated. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "SELECT v FROM OPENJSON(N'{\"a\":1}') WITH (v int 'strict $.missing');", + "expect": { + "error": 13608 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A missing property is an error under strict and NULL in lax, and lax is the default nobody writes. This is the number that tells the two modes apart. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "DECLARE @j json = JSON_OBJECT('big': REPLICATE(CAST(N'x' AS nvarchar(max)), 5000));\nSELECT JSON_VALUE(@j, 'strict $.big') AS x;", + "expect": { + "error": 13625 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The 4000 character cliff, made loud. JSON_VALUE without RETURNING is typed nvarchar(4000) and strict mode is the only setting that says so. It goes red if the statement succeeds, or fails with any other number." + }, + { + "sql": "DECLARE @j json = JSON_OBJECT('big': REPLICATE(CAST(N'x' AS nvarchar(max)), 5000));\nSELECT CASE WHEN JSON_VALUE(@j, '$.big') IS NULL THEN 1 ELSE 0 END AS silent_null;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The half of the cliff that is dangerous, and the reason the skill calls it data loss dressed as a missing key: in lax mode, which is the default, the same value comes back NULL with no error at all. It goes red if the value ever starts arriving whole or starts raising instead." + } + , + { + "sql": "ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 120;\nGO\nSELECT COUNT(*) AS n FROM OPENJSON(N'[1,2]');", + "expect": { + "error": 208 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. OPENJSON is hidden below compatibility level 130 and reports as a missing object rather than as an unavailable feature, which sends a reader looking for a spelling mistake. This probe is last in the skill's set on purpose: it lowers the compatibility level of the scratch database the lane drops afterwards, so nothing later in this skill depends on the level it leaves behind. It goes red if the statement succeeds, or fails with any other number." + } + ] + }, + "triggering": { + "explicit": [ + "Store this API payload as JSON in Azure SQL Database and query it", + "Shred this JSON array into rows with OPENJSON", + "Should this document column be nvarchar(max) or the json type", + "Add an index so this JSON property lookup stops scanning the table", + "Flatten this nested payload into a result set with typed columns", + "Review this OPENJSON query before it ships" + ], + "implicit": [ + "JSON_VALUE returns NULL but the property is right there in the document", + "invalid JSON got into the table and nothing stopped it", + "how do I filter on a field inside the JSON column", + "parse the response we saved from the vendor and load it into a table", + "my JSON query got slower as the table grew", + "the data types json and nvarchar are incompatible in the equal to operator", + "the JSON data type cannot be compared or sorted", + "why did the whitespace and one of my duplicate keys disappear" + ], + "routing": [ + "we save webhook payloads whole, how should we read them back", + "turn this document column into something reportable" + ], + "negative": [ + "Write me an upsert that inserts or updates a row", + "Write a MERGE to sync this staging table into the target", + "Where should the json column go in this table design and how wide is the row", + "How long should the string columns in my index be", + "Write me a paginated query for Azure SQL", + "How do I get the id of the row I just inserted", + "Store these embeddings and run a similarity search", + "Parse this JSON file in Python before loading it" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge deleted file mode 120000 index 691abda..0000000 --- a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge +++ /dev/null @@ -1 +0,0 @@ -../../../skills/t-sql-upserts-merge \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/SKILL.md b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/SKILL.md new file mode 100644 index 0000000..fb29418 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/SKILL.md @@ -0,0 +1,231 @@ +--- +name: t-sql-upserts-merge +description: >- + Writes an upsert for Azure SQL Database that is still correct when two sessions run it at the + same moment, and refuses the MERGE shapes that lose rows. Use when asked to "insert or update", + "insert if not exists", "add or update", "make this insert idempotent", "upsert", "write a + MERGE", "sync a staging table into the target table", make a row-by-row load or import safe to + run twice, or port ON CONFLICT DO UPDATE or ON DUPLICATE KEY UPDATE; and use when duplicate rows + appear that nothing in the application created, or when error 2627, 2601, 8672, 10713 or a + deadlock shows up under load. Covers why IF EXISTS then UPDATE ELSE INSERT is a race, what MERGE + needs to be safe, which MERGE shapes to refuse outright, and the two patterns that are safe + without MERGE. Key and index design belongs to design-azure-sql-schema, and general T-SQL + dialect to t-sql-correctness. +--- + +# Write an upsert that survives concurrency, and refuse the dangerous MERGE + +An upsert looks like a one-statement problem and is a concurrency problem. Both of the shapes a +model reaches for first, `IF EXISTS ... UPDATE ELSE INSERT` and a bare `MERGE`, are races. They +pass every single-session test and fail the first time two callers arrive together. + +Measured on 2026-08-28 against a live engine reporting `EngineEdition` 5 and Edition `SQL Azure`. +Open [the measured runs](references/merge-hazards-and-concurrency.md) when a number below is +disputed, or before changing one. + +## What the race actually costs + +Eight sessions, twenty increments each, one hot key. A correct upsert ends at 160. + +| Pattern | Result | Cost | +|---|---|---| +| `UPDATE` then `INSERT` when `@@ROWCOUNT = 0`, no hint | one session died on `Msg 2627`, final value **140** | 20 writes lost; the caller saw one error, not twenty | +| `MERGE` with no lock hint, 4 sessions | 2 of 4 raised `Msg 2627`, final value **2** of 4 | half the batch rolled back | +| `IF EXISTS` on a table with **no unique key**, 2 sessions | **2 rows for one key, no error at all** | silent duplicates, nothing in the log to find later | +| `MERGE WITH (HOLDLOCK)` on a unique key | 160, no failures, no deadlocks | correct | +| `UPDATE WITH (SERIALIZABLE)` then `INSERT` | 160, no failures | correct | +| `UPDATE`, then `INSERT` inside `TRY` catching 2627 | 160, no failures | correct | + +The third row is the one to fear. The others raise an error someone can see. That one returns +success to both callers and leaves the table wrong. + +## Why the naive shape is a race + +```sql +-- WRONG. Two sessions can both take the ELSE branch. +IF EXISTS (SELECT 1 FROM dbo.counters WHERE k = @k) + UPDATE dbo.counters SET n = n + 1 WHERE k = @k; +ELSE + INSERT INTO dbo.counters (k, n) VALUES (@k, 1); +``` + +The `SELECT` takes a shared lock and releases it immediately, under every isolation level below +serializable, so there is a window between the test and the insert. `BEGIN TRAN` does not close +it: a transaction makes the pair atomic on failure without making the read block anyone. +Verified with a two second gap in the window, both sessions reported `INSERT branch`. + +## Three upserts that hold + +Each reached 160 of 160 under eight concurrent sessions. + +**1. MERGE, when the statement genuinely needs more than one action.** + +```sql +BEGIN TRAN; +MERGE dbo.counters WITH (HOLDLOCK) AS t +USING (SELECT @k AS k) AS s + ON t.k = s.k +WHEN MATCHED THEN + UPDATE SET t.n = t.n + 1 +WHEN NOT MATCHED THEN + INSERT (k, n) VALUES (s.k, 1); +COMMIT; +``` + +The hint goes on the target, before the alias. Two things make it safe and both are required. +`HOLDLOCK` is documented as a synonym for `SERIALIZABLE`, and it is what holds a range lock across +the match and the insert. **The unique index on the join key is what makes that range lock +narrow.** Without it the lock covers far more of the table: four concurrent sessions against a +table with no unique key produced `Msg 1205` deadlocks and rolled two of them back. + +**2. UPDATE first, insert only if nothing was updated.** Fewer moving parts, and the better +choice when the row usually already exists. + +```sql +BEGIN TRAN; +UPDATE dbo.counters WITH (SERIALIZABLE) SET n = n + 1 WHERE k = @k; +IF @@ROWCOUNT = 0 + INSERT INTO dbo.counters (k, n) VALUES (@k, 1); +COMMIT; +``` + +`SERIALIZABLE` on the `UPDATE` is not decoration. Without it this is the row that ended at 140. + +**3. Catch the duplicate key and treat it as the update signal.** Best when the row usually does +not exist, because the common path takes no extra lock. + +```sql +UPDATE dbo.counters SET n = n + 1 WHERE k = @k; +IF @@ROWCOUNT = 0 +BEGIN + BEGIN TRY + INSERT INTO dbo.counters (k, n) VALUES (@k, 1); + END TRY + BEGIN CATCH + IF ERROR_NUMBER() IN (2601, 2627) + UPDATE dbo.counters SET n = n + 1 WHERE k = @k; + ELSE + THROW; + END CATCH +END +``` + +All three still depend on a unique constraint or unique index on the key being upserted. **Without +one, none of them is safe**, because there is nothing for the range lock to hold and nothing to +raise 2627 against. Sizing that key is `design-azure-sql-schema`. + +## When to refuse MERGE outright + +Say no and write separate statements. Each is verified or documented, not folklore. + +- **A partial batch with `WHEN NOT MATCHED BY SOURCE THEN DELETE`.** That clause belongs to a full + reload, never to an upsert of whichever rows this request happens to carry. Verified: a target + holding `A`, `B`, `C` and a source naming only `A` ended with one row and no error. +- **A source that can contain the same key twice.** `Msg 8672`, "The MERGE statement attempted to + UPDATE or DELETE the same row more than once". Deduplicate the source first, with `GROUP BY` or + a window function, rather than loosening the `ON` clause. +- **Any filter in the `ON` clause.** Documented: put only the columns being matched there. + Verified: adding `AND t.qty > 100` to `ON` made an existing row look unmatched, so the + `NOT MATCHED` branch fired and hit `Msg 2627` on the key that already existed. Row filters go in + the `WHEN` clause or in a view or common table expression over the source. +- **A columnstore target.** Documented: avoid any form of columnstore index as a `MERGE` target, + and instead stage, then delete and insert in batches. +- **High concurrency at scale.** Documented: at scale `MERGE` may introduce complicated + concurrency issues, and separate statements can perform better with less blocking. +- **Literal values in a frequently executed statement.** Simple parameterization is not applied to + `MERGE`, so literals compile a new plan every execution. Pass parameters through + `sys.sp_executesql`. Building the statement by concatenation instead is `prevent-sql-injection`. + +`MERGE` also requires a semicolon terminator, and omitting it is `Msg 10713` by number rather than +a generic syntax error. + +## The indexed view question, stated honestly + +A long standing warning says a `MERGE` can leave an indexed view inconsistent with its base +tables, and a routine consistency check will not report it. Half of that is current. + +**The half that holds.** An integrity check performs only **physical** consistency checks on an +indexed view. The logical checks, which compare the materialized rows against the base tables, run +only under `EXTENDED_LOGICAL_CHECKS`. So a view that disagrees with its base tables is invisible to +a routine check, whatever put it there. Documented for Azure SQL Database today, and why this +defect class was ever severe. + +**The half that did not reproduce.** The published minimal reproduction and four further shapes, +including an aggregated view over a two table join, stayed consistent at compatibility levels 120 +through 170. So do not tell a developer their indexed views are being corrupted today. Say instead +that this defect raises no error and fails no check, so an indexed view over a `MERGE` target has +to be verified deliberately. The repair, if the two ever disagree, is to drop and recreate the +view's clustered index. + +## Check it worked + +One session proves nothing, so run the upsert from two shells at once and compare the final value +against the arithmetic total. `sqlcmd -b` exits non-zero on an error, and `-m-1` prints +every message, including the severity 10 ones that otherwise arrive with no number at all. + +**`-m-1` is an ODBC `sqlcmd` instruction**, meaning the 18.x build from `mssql-tools18` or the +Microsoft command line utilities, and `localhost,1433` is exactly where a reader is most likely to +be holding the other one. Measured 2026-09-05, go-sqlcmd 1.10.0, the 1.x build +`brew install sqlcmd` and `winget install sqlcmd` install, prints no `Msg` header on a severity 10 +message at any `-m` value. `Msg 2627` below is severity 14 and arrives with its number on either +build, so this check still works; what go-sqlcmd will not show you is a severity 10 message from the +statements around it. The ODBC build is inside the container image at +`/opt/mssql-tools18/bin/sqlcmd`, one `docker exec` away. +`build-app-on-azure-sql` tells the two builds apart in one table. + +```bash +UPSERT="BEGIN TRAN; +UPDATE dbo.counters WITH (SERIALIZABLE) SET n = n + 1 WHERE k = N'hot'; +IF @@ROWCOUNT = 0 INSERT INTO dbo.counters (k, n) VALUES (N'hot', 1); +COMMIT;" +S=(-S localhost,1433 -d appdb -C -b -m-1) + +sqlcmd "${S[@]}" -Q "CREATE TABLE dbo.counters (k NVARCHAR(50) NOT NULL PRIMARY KEY, n INT NOT NULL);" + +for i in 1 2; do + sqlcmd "${S[@]}" -Q "DECLARE @i INT = 0; WHILE @i < 50 BEGIN $UPSERT SET @i += 1; END" & +done +wait + +sqlcmd "${S[@]}" -Q "SELECT n, COUNT(*) OVER () AS rows_for_key FROM dbo.counters WHERE k = N'hot';" +``` + +Expected: `n` is 100 and `rows_for_key` is 1. Below 100 means writes were lost, more than one row +means the key carries no unique index. Delete `WITH (SERIALIZABLE)` and rerun: the count falls +short, or one shell exits non-zero on `Msg 2627`. Against Azure SQL Database only `-S` and the +authentication flags change. + +Check an indexed view over the target separately, because nothing else will tell you. + +```sql +-- Cheapest first: read the materialized rows, then recompute from the base table. +SELECT region, total, n FROM dbo.v_totals WITH (NOEXPAND) ORDER BY region; +SELECT region, SUM(amt), COUNT_BIG(*) FROM dbo.sales GROUP BY region ORDER BY region; + +-- Or ask for the logical checks by name. Without this option they do not run. +DBCC CHECKTABLE ('dbo.v_totals') WITH EXTENDED_LOGICAL_CHECKS; +``` + +The two result sets must match row for row. `DBCC` must report 0 allocation and 0 consistency +errors. + +## Do not + +- Do not present `IF EXISTS ... UPDATE ELSE INSERT` as an upsert. It is a race, and wrapping it in + a transaction does not fix it. +- Do not add `HOLDLOCK` and stop there. Without a unique index on the join key the range lock is + wide, and four sessions produced `Msg 1205`. +- Do not translate `ON CONFLICT DO UPDATE` into `MERGE` reflexively. Two of the three safe + patterns are plain statements and both are easier to review. +- Do not use `READPAST` on a `MERGE` target with an insert branch. Documented: it can produce + inserts that violate unique constraints. +- Do not claim an indexed view is being corrupted by `MERGE` today without reproducing it, and do + not treat a clean integrity check as evidence that one agrees with its base tables. +- Do not design the retry loop here. A deadlock victim needs a retry rather than a louder + message, and which errors deserve one is separate work. + +## References + +- Open [the measured runs](references/merge-hazards-and-concurrency.md) when a number here is + challenged, or when you need the message text for 2627, 8672, 10713 or 1205 as this engine + printed it. Every concurrency run, every loud `MERGE` shape, all five indexed view attempts. diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/references/merge-hazards-and-concurrency.md b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/references/merge-hazards-and-concurrency.md new file mode 100644 index 0000000..545749c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/references/merge-hazards-and-concurrency.md @@ -0,0 +1,267 @@ +# MERGE hazards and upsert concurrency: the runs behind every number in SKILL.md + +## Contents + +- [How this was measured](#how-this-was-measured) +- [The naive upsert races](#the-naive-upsert-races) +- [MERGE without a lock hint](#merge-without-a-lock-hint) +- [MERGE with HOLDLOCK](#merge-with-holdlock) +- [HOLDLOCK without a unique index on the join key](#holdlock-without-a-unique-index-on-the-join-key) +- [The three patterns that reached 160 of 160](#the-three-patterns-that-reached-160-of-160) +- [MERGE shapes that fail loudly](#merge-shapes-that-fail-loudly) +- [WHEN NOT MATCHED BY SOURCE THEN DELETE](#when-not-matched-by-source-then-delete) +- [The indexed view claim](#the-indexed-view-claim) +- [What is documented rather than measured](#what-is-documented-rather-than-measured) + +## How this was measured + +One engine, reachable over the network, reporting: + +```text +SERVERPROPERTY('EngineEdition') = 5 +SERVERPROPERTY('Edition') = SQL Azure +``` + +Each concurrency run started separate client processes against the same database, one process per +session, and compared the final value against the arithmetic total. Date of the run: 2026-08-28. + +Two tables carried the tests: + +```sql +CREATE TABLE dbo.counter_pk (k NVARCHAR(50) NOT NULL PRIMARY KEY, n INT NOT NULL); +CREATE TABLE dbo.counter_nokey (k NVARCHAR(50) NOT NULL, n INT NOT NULL); +``` + +## The naive upsert races + +Two sessions, a two second delay inserted between the existence test and the write so the window +is deterministic rather than lucky. + +```sql +BEGIN TRAN; +IF EXISTS (SELECT 1 FROM dbo.counter_pk WHERE k = @k) +BEGIN WAITFOR DELAY '00:00:02'; UPDATE dbo.counter_pk SET n = n + 1 WHERE k = @k; END +ELSE +BEGIN WAITFOR DELAY '00:00:02'; INSERT INTO dbo.counter_pk (k, n) VALUES (@k, 1); END +COMMIT; +``` + +Both sessions printed `took INSERT branch`. One of them then failed: + +```text +Msg 2627, Level 14, State 1, Server SQL Azure, Line 13 +Violation of PRIMARY KEY constraint 'PK__counter___...'. Cannot insert duplicate key in object +'dbo.counter_pk'. The duplicate key value is (race1). +The statement has been terminated. +``` + +Final state: one row, `n = 1`. One of the two increments was lost. + +The same code against `dbo.counter_nokey`, which has no unique key: + +```text +INSERT branch +INSERT branch +DUPLICATE ROWS for race2: 2 +``` + +**No error was raised.** Both callers were told the write succeeded, and the table holds two rows +for one logical key. This is the shape that does not show up in a log. + +## MERGE without a lock hint + +Four concurrent sessions, one key, target has a primary key: + +```sql +MERGE dbo.counter_pk AS t +USING (SELECT @k AS k, 1 AS n) AS s ON t.k = s.k +WHEN MATCHED THEN UPDATE SET t.n = t.n + 1 +WHEN NOT MATCHED THEN INSERT (k, n) VALUES (s.k, s.n); +``` + +Two sessions raised `Msg 2627` on the same constraint. Final value `n = 2` where four increments +were requested. + +## MERGE with HOLDLOCK + +Identical statement, one hint added: + +```sql +MERGE dbo.counter_pk WITH (HOLDLOCK) AS t +USING (SELECT @k AS k, 1 AS n) AS s ON t.k = s.k +WHEN MATCHED THEN UPDATE SET t.n = t.n + 1 +WHEN NOT MATCHED THEN INSERT (k, n) VALUES (s.k, s.n); +``` + +Four sessions: all four succeeded, `n = 4`, one row. + +Eight sessions running twenty iterations each: all eight printed `session finished 20 iterations`, +final `n = 160` against an expected 160, one row, no deadlocks. + +## HOLDLOCK without a unique index on the join key + +The same `MERGE WITH (HOLDLOCK)` against `dbo.counter_nokey`, four sessions: + +```text +Msg 1205, Level 13, State 18, Server SQL Azure, Line 4 +Transaction (Process ID 88) was deadlocked on lock resources with another process and has been +chosen as the deadlock victim. Rerun the transaction. +``` + +Two of the four were chosen as victims and rolled back. The two survivors left one row with +`n = 2`. So the hint still serialized the work, but at the cost of a lock wide enough to deadlock, +and half the transactions were lost rather than duplicated. + +The unique index is what turns that into a narrow key range lock. It is not optional. + +## The three patterns that reached 160 of 160 + +Eight sessions, twenty iterations each, one hot key. + +| Pattern | Outcome | +|---|---| +| `MERGE WITH (HOLDLOCK)` | 8 of 8 finished, `n = 160` | +| `UPDATE WITH (SERIALIZABLE)` then `INSERT` when `@@ROWCOUNT = 0` | 8 of 8 finished, `n = 160` | +| `UPDATE`, then `INSERT` inside `TRY` catching 2601 and 2627 | 8 of 8 finished, `n = 160` | +| `UPDATE` then `INSERT` with **no** hint | 7 of 8 finished, one raised `Msg 2627`, `n = 140` | + +The last row is the measurement worth carrying: one duplicate key error at one moment cost the +whole remaining batch for that session, twenty writes, because the error terminated it. + +## MERGE shapes that fail loudly + +Target `dbo.prod (sku NVARCHAR(20) PRIMARY KEY, qty INT)` holding `A`, `B`, `C`. + +**Duplicate keys in the source:** + +```sql +MERGE dbo.prod AS t +USING (VALUES (N'A', 10), (N'A', 20)) AS s(sku, qty) ON t.sku = s.sku +WHEN MATCHED THEN UPDATE SET t.qty = s.qty +WHEN NOT MATCHED THEN INSERT (sku, qty) VALUES (s.sku, s.qty); +``` + +```text +Msg 8672, Level 16, State 1 +The MERGE statement attempted to UPDATE or DELETE the same row more than once. This happens when a +target row matches more than one source row. A MERGE statement cannot UPDATE/DELETE the same row of +the target table multiple times. Refine the ON clause to ensure a target row matches at most one +source row, or use the GROUP BY clause to group the source rows. +``` + +**Missing terminator:** + +```text +Msg 10713, Level 15, State 1 +A MERGE statement must be terminated by a semi-colon (;). +``` + +**A filter in the ON clause**, `ON t.sku = s.sku AND t.qty > 100`, against a row that exists with +`qty` below 100: + +```text +Msg 2627, Level 14, State 1 +Violation of PRIMARY KEY constraint 'PK__prod__...'. Cannot insert duplicate key in object +'dbo.prod'. The duplicate key value is (A). +``` + +The row exists, so the intent was an update, but the extra condition made it unmatched and the +insert branch fired. This is the documented "unexpected and incorrect results" behaviour, and here +it surfaced as a constraint violation rather than as wrong data, because the key was protected. +On a table with no unique key it would have inserted a duplicate instead. + +## WHEN NOT MATCHED BY SOURCE THEN DELETE + +The single most destructive shape, and the easiest to add by accident, because it makes a `MERGE` +look complete. + +```sql +-- dbo.prod holds A, B, C. The batch mentions only A. +MERGE dbo.prod AS t +USING (VALUES (N'A', 5)) AS s(sku, qty) ON t.sku = s.sku +WHEN MATCHED THEN UPDATE SET t.qty = s.qty +WHEN NOT MATCHED BY TARGET THEN INSERT (sku, qty) VALUES (s.sku, s.qty) +WHEN NOT MATCHED BY SOURCE THEN DELETE; +``` + +```text +before: A,B,C +after : A +rows left = 1 +``` + +Two rows deleted, no error, no list of what went. The clause is correct only when the source is +the complete intended contents of the target. + +## The indexed view claim + +The claim under test: a `MERGE` can leave an indexed view inconsistent with its base tables, and +a routine consistency check will not report it. + +**Attempt 1, the published minimal reproduction.** A single table, a schemabound view over two of +its columns with a unique clustered index, and a `MERGE` combining `WHEN MATCHED THEN UPDATE` on a +column outside the view with `WHEN NOT MATCHED BY SOURCE THEN DELETE`. Run at compatibility levels +120, 130, 140, 150, 160 and 170: + +```text +compat 120: base=1 view=1 consistent +compat 130: base=1 view=1 consistent +compat 140: base=1 view=1 consistent +compat 150: base=1 view=1 consistent +compat 160: base=1 view=1 consistent +compat 170: base=1 view=1 consistent +``` + +**Attempt 2, the foreign key join elimination shape.** A parent and a child table, a single column +foreign key from child to parent with no cascade, a schemabound view joining the two with a unique +clustered index on the child key, and a `MERGE` on the child combining `UPDATE` with +`NOT MATCHED BY SOURCE THEN DELETE`. The view and the recomputation agreed, before and after. + +**Attempt 3, the same join with aggregation.** `SUM` and `COUNT_BIG(*)` grouped by the parent key +over the same foreign key join. Agreed. + +**Attempt 4, a matched delete.** `WHEN MATCHED AND THEN DELETE` alongside +`WHEN MATCHED THEN UPDATE`, and separately all three actions in one statement. Agreed. + +**Attempt 5, a system versioned target.** A `MERGE` combining insert, update and delete against a +system versioned table carrying a nonclustered index on its history table, another open report in +the same family, completed and produced the expected current rows and history rows. + +**Conclusion.** The corruption did not reproduce on this engine in any shape tried. Treat the +catalog note as describing a class of defect rather than current behaviour. + +**What does still hold**, and it is what made the class severe: an integrity check performs only +physical consistency checks on an indexed view by default. The logical checks that compare the +materialized rows against the base tables run only under `EXTENDED_LOGICAL_CHECKS`. Both forms +were run here and both are accepted on this engine: + +```sql +DBCC CHECKTABLE ('dbo.v_totals'); +DBCC CHECKTABLE ('dbo.v_totals') WITH EXTENDED_LOGICAL_CHECKS; +DBCC CHECKDB; +DBCC CHECKDB WITH EXTENDED_LOGICAL_CHECKS; +``` + +The cheaper habit is to compare the view read with `NOEXPAND` against a recomputation from the +base tables, which needs no elevated permission and is a plain query. + +## What is documented rather than measured + +These come from the Microsoft Learn pages for `MERGE` and for the database consistency check, +read on 2026-08-28 for Azure SQL Database. They were not reproduced here. + +- `HOLDLOCK` is called out as the way to prevent unique key violations where a `MERGE` both + inserts and updates unique keys, and it is a synonym for `SERIALIZABLE`. +- The index guidance is a unique index on the join columns of the source and a unique clustered + index on the join columns of the target. +- Only the matching columns belong in `ON`. Comparisons to other values, including constants, can + return unexpected and incorrect results. Filters belong in a `WHEN` clause or a view. +- `READPAST` with an insert branch can produce inserts that violate unique constraints. +- Tables with any form of columnstore index are called out as poor `MERGE` targets. +- Simple parameterization is not applied to `MERGE`, so literals force a compile per execution. +- At scale, `MERGE` may introduce complicated concurrency issues, and discrete statements can + block less. +- Inside an `AFTER` trigger, `@@ROWCOUNT` reflects the total rows affected by the whole `MERGE`, + not just the action the trigger was declared for. +- The consistency check is unavailable on the Hyperscale service tier, where a per-table check is + the documented substitute. diff --git a/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/skill.spec.jsonc new file mode 100644 index 0000000..a95818c --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/t-sql-upserts-merge/skill.spec.jsonc @@ -0,0 +1,186 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "t-sql-upserts-merge", + "domain": "tsql-correctness-and-patterns", + "value": ["corrects-a-wrong-assumption", "multi-step-with-silent-failures"], + "correction": "Asked for an upsert on Azure SQL Database, an agent writes IF EXISTS then UPDATE ELSE INSERT, or a bare MERGE, and presents either as finished, when both are races that pass every single-session test and lose writes the first time two callers arrive together. Measured on 2026-08-28 against a live engine reporting EngineEdition 5 and Edition 'SQL Azure': eight sessions running twenty increments each on one key ended at 140 of 160 for the unhinted UPDATE-then-INSERT pattern, because one duplicate key error terminated a whole session's batch, while MERGE WITH (HOLDLOCK), UPDATE WITH (SERIALIZABLE) then INSERT, and UPDATE then INSERT catching 2601 and 2627 all reached 160; four concurrent unhinted MERGE statements raised Msg 2627 on two of them and applied two of four increments; and the naive shape against a table with no unique key produced two rows for one key with no error raised to either caller, which is the failure nothing later can find. HOLDLOCK alone is not the fix either, since without a unique index on the join key the range lock is wide enough that four sessions produced Msg 1205 and two were rolled back. The other MERGE shapes worth refusing are measured too: a filter added to the ON clause made an existing row look unmatched, so the NOT MATCHED branch fired and hit Msg 2627 on a key that already existed. The catalog note that indexed view corruption from MERGE is invisible to a consistency check is half right and must be stated as such: the published minimal reproduction and four further shapes, including a two table view over a single column foreign key and an aggregated view over the same join, all stayed consistent at compatibility levels 120 through 170, so the corruption did not reproduce; but the reason it was severe is documented and current, since a consistency check performs only physical checks on an indexed view unless EXTENDED_LOGICAL_CHECKS is requested, so any disagreement between a materialized view and its base tables is invisible to a routine check whatever caused it.", + "value_probe": { + "gate": "The answer makes the existence check and the write atomic with respect to concurrent writers, by any route: a locking hint on a single statement, a serializable transaction around the pair, or catching the duplicate key error and retrying. An answer that leaves the check and the write as two statements another session can interleave has not reached it.", + "task": "We have an Azure SQL Database table dbo.counters (k nvarchar(100), n int) that tracks a hit count per key. Our API increments a counter on every request, so many application instances call this at the same moment for the same key.\n\nRight now we do this and we are seeing counts come out lower than the number of requests:\n\n```sql\nIF EXISTS (SELECT 1 FROM dbo.counters WHERE k = @k)\n UPDATE dbo.counters SET n = n + 1 WHERE k = @k;\nELSE\n INSERT INTO dbo.counters (k, n) VALUES (@k, 1);\n```\n\nWrite the correct version. Give me the SQL I should actually run, and tell me anything else I need to do to make it correct. Be specific and concrete.", + "states": [ + "A unique constraint or unique index on the key column is required for the fix to work, not merely recommended or helpful.", + "Without a unique key on the column, concurrent callers can produce two rows for the same key and no error is raised to either caller.", + "The check and the write must be made one atomic operation with respect to other writers, rather than left as two statements that another session can interleave." + ], + "refutes": [ + "MERGE is unsafe or unreliable for concurrent upserts on Azure SQL Database even when a locking hint and a unique key are used, and should therefore be avoided for this problem.", + "Wrapping the existing IF EXISTS then UPDATE ELSE INSERT in a transaction, with no locking hint and no unique key, is sufficient to make it correct under concurrency." + ] + }, + "posture": ["read", "write"], + // Demoted from preview to draft on 2026-09-20. The reading of 2026-09-06 rested + // on three runs per arm, so it was re-measured at ten before this field moved. + // UNCLEAR, and nothing moved in the skill's favour: the gate reads 10 of 10 in + // both arms, and the statement that the check and the write have to be one atomic + // operation falls to 8 of 10 with the skill from 10 of 10 without. The model + // already answers this skill's own task correctly without it. + // This is an author declaring below the ceiling, which the ladder permits, and + // it is not the skill failing. Its engine probes still pass and nothing here is + // unverified. Do not raise this field to clear the promotion backlog. It moves + // when the skill is rewritten to say something the model does not already know + // and a fresh measurement shows lift, and not before. + "maturity": "draft", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "t-sql-correctness", + "t-sql-json-and-openjson", + "design-azure-sql-schema", + "t-sql-error-handling", + "t-sql-programmability-objects", + "bulk-load-and-bulk-copy", + "diagnose-blocking-and-deadlocks", + "prevent-sql-injection" + ], + "validation": { + "target": "both", + "setup": [ + "a table with a unique constraint or unique index on the key being upserted", + "a second table holding the same shape with no unique key at all", + "the ability to open at least four client sessions against the same database concurrently", + "a schemabound view over one of those tables carrying a unique clustered index" + ], + "assert": [ + "the generated upsert holds a lock across the match and the write, whether by HOLDLOCK on a MERGE, SERIALIZABLE on the leading UPDATE, or a caught 2601 or 2627", + "no generated upsert is a bare IF EXISTS then UPDATE ELSE INSERT, with or without a surrounding transaction", + "every generated MERGE names a target that carries a unique index on the join key, and the answer says so when it does not", + "the ON clause of every generated MERGE contains only columns compared between source and target, and no constant or filter", + "every generated MERGE ends with a semicolon", + "running the generated upsert from four concurrent sessions against one key leaves exactly one row and applies every increment", + "running it against the table with no unique key is refused or flagged rather than silently producing duplicate rows", + "no answer claims a MERGE is corrupting an indexed view without a reproduction, and no answer offers a clean consistency check as evidence that a materialized view agrees with its base tables" + ], + + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_merge_semi (sku NVARCHAR(20) NOT NULL PRIMARY KEY, qty INT NOT NULL);\nGO\nMERGE dbo.probe_merge_semi WITH (HOLDLOCK) AS t\nUSING (VALUES (N'A', 1)) AS s(sku, qty) ON t.sku = s.sku\nWHEN MATCHED THEN UPDATE SET t.qty = s.qty\nWHEN NOT MATCHED THEN INSERT (sku, qty) VALUES (s.sku, s.qty)", + "expect": { + "error": 10713 + }, + "why": "The skill claims a missing terminator is Msg 10713 by number rather than a generic syntax error, which is what makes it findable. Microsoft Learn states it by number on the MERGE page: 'Error 10713 is raised when a MERGE statement is run without the terminator.' Measured 2026-08-28. The CREATE TABLE sits in its own batch before GO so the failure is unambiguously the MERGE." + }, + { + "sql": "CREATE TABLE dbo.probe_merge_dupsrc (sku NVARCHAR(20) NOT NULL PRIMARY KEY, qty INT NOT NULL);\nINSERT INTO dbo.probe_merge_dupsrc (sku, qty) VALUES (N'A', 1);\nGO\nMERGE dbo.probe_merge_dupsrc WITH (HOLDLOCK) AS t\nUSING (VALUES (N'A', 10), (N'A', 20)) AS s(sku, qty) ON t.sku = s.sku\nWHEN MATCHED THEN UPDATE SET t.qty = s.qty\nWHEN NOT MATCHED THEN INSERT (sku, qty) VALUES (s.sku, s.qty);", + "expect": { + "error": 8672 + }, + "why": "The refuse list says a source that can carry one key twice is Msg 8672, and this is that claim made runnable. Learn states the rule without the number ('when UPDATE is specified and more than one row of table_source matches a row in target_table, SQL Server returns an error'); the number 8672 comes from this skill's own run on 2026-08-28. HOLDLOCK is present so the probe isolates the duplicate source rather than a locking difference." + }, + { + "sql": "CREATE TABLE dbo.probe_merge_onfilter (sku NVARCHAR(20) NOT NULL PRIMARY KEY, qty INT NOT NULL);\nINSERT INTO dbo.probe_merge_onfilter (sku, qty) VALUES (N'A', 10);\nGO\nMERGE dbo.probe_merge_onfilter WITH (HOLDLOCK) AS t\nUSING (VALUES (N'A', 20)) AS s(sku, qty) ON t.sku = s.sku AND t.qty > 100\nWHEN MATCHED THEN UPDATE SET t.qty = s.qty\nWHEN NOT MATCHED THEN INSERT (sku, qty) VALUES (s.sku, s.qty);", + "expect": { + "error": 2627 + }, + "why": "The single most useful probe here, because it turns a documented warning into a number. Learn says only matching columns belong in ON and that filtering there 'can return unexpected and incorrect results', with no error and no example. The row for A exists, the extra AND t.qty > 100 makes it look unmatched, the NOT MATCHED branch fires, and the primary key refuses it with Msg 2627. Measured 2026-08-28. On a target with no unique key the same shape inserts a duplicate silently instead, which is the failure this skill exists to stop." + }, + { + "sql": "CREATE TABLE dbo.probe_merge_delsource (sku NVARCHAR(20) NOT NULL PRIMARY KEY, qty INT NOT NULL);\nINSERT INTO dbo.probe_merge_delsource (sku, qty) VALUES (N'A', 1), (N'B', 1), (N'C', 1);\nGO\nMERGE dbo.probe_merge_delsource WITH (HOLDLOCK) AS t\nUSING (VALUES (N'A', 5)) AS s(sku, qty) ON t.sku = s.sku\nWHEN MATCHED THEN UPDATE SET t.qty = s.qty\nWHEN NOT MATCHED BY TARGET THEN INSERT (sku, qty) VALUES (s.sku, s.qty)\nWHEN NOT MATCHED BY SOURCE THEN DELETE;\nSELECT COUNT(*) AS rows_left FROM dbo.probe_merge_delsource;", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "The first entry in the refuse list, made executable. Three rows in, a source naming one of them, and two rows gone with no error and no list of what went. Measured 2026-08-28. This is the shape an agent adds because it makes a MERGE look complete, and the probe asserts the damage rather than the intent." + }, + { + "sql": "CREATE TABLE dbo.probe_upsert_nokey (k NVARCHAR(50) NOT NULL, n INT NOT NULL);\nINSERT INTO dbo.probe_upsert_nokey (k, n) VALUES (N'hot', 1);\nINSERT INTO dbo.probe_upsert_nokey (k, n) VALUES (N'hot', 1);\nSELECT COUNT(*) AS rows_for_one_key FROM dbo.probe_upsert_nokey WHERE k = N'hot';", + "expect": { + "rows": [ + [ + 2 + ] + ] + }, + "why": "Why every pattern in this skill is conditional on a unique key. Two writes of the same key against a table without one are both accepted, so the table ends with two rows for one logical key and neither caller was told anything. That is the mechanism behind the measured two-session result in the body table, reduced to the part a single session can prove: there is nothing here to raise 2627 and nothing for a range lock to hold. Measured 2026-08-28." + }, + { + "sql": "CREATE TABLE dbo.probe_merge_holdlock (k NVARCHAR(50) NOT NULL PRIMARY KEY, n INT NOT NULL);\nGO\nMERGE dbo.probe_merge_holdlock WITH (HOLDLOCK) AS t\nUSING (SELECT N'hot' AS k) AS s ON t.k = s.k\nWHEN MATCHED THEN UPDATE SET t.n = t.n + 1\nWHEN NOT MATCHED THEN INSERT (k, n) VALUES (s.k, 1);\nGO\nMERGE dbo.probe_merge_holdlock WITH (HOLDLOCK) AS t\nUSING (SELECT N'hot' AS k) AS s ON t.k = s.k\nWHEN MATCHED THEN UPDATE SET t.n = t.n + 1\nWHEN NOT MATCHED THEN INSERT (k, n) VALUES (s.k, 1);\nGO\nSELECT n, COUNT(*) OVER () AS rows_for_key FROM dbo.probe_merge_holdlock WHERE k = N'hot';", + "expect": { + "rows": [ + [ + 2, + 1 + ] + ] + }, + "why": "The recommended statement, run twice, asserting that the hint parses where the skill puts it and that the branch selection is right: the first run inserts, the second updates, one row and n = 2. Learn's syntax places the hint on the target before the alias, target_table [WITH (merge_hint)] [[AS] table_alias], and this is the only probe that proves the skill's own code block compiles. A single session cannot prove the concurrency claim, which is what the two-shell check in Check it worked is for." + }, + { + "sql": "CREATE TABLE dbo.probe_upsert_serializable (k NVARCHAR(50) NOT NULL PRIMARY KEY, n INT NOT NULL);\nGO\nBEGIN TRAN;\nUPDATE dbo.probe_upsert_serializable WITH (SERIALIZABLE) SET n = n + 1 WHERE k = N'hot';\nIF @@ROWCOUNT = 0 INSERT INTO dbo.probe_upsert_serializable (k, n) VALUES (N'hot', 1);\nCOMMIT;\nGO\nBEGIN TRAN;\nUPDATE dbo.probe_upsert_serializable WITH (SERIALIZABLE) SET n = n + 1 WHERE k = N'hot';\nIF @@ROWCOUNT = 0 INSERT INTO dbo.probe_upsert_serializable (k, n) VALUES (N'hot', 1);\nCOMMIT;\nGO\nSELECT n, COUNT(*) OVER () AS rows_for_key FROM dbo.probe_upsert_serializable WHERE k = N'hot';", + "expect": { + "rows": [ + [ + 2, + 1 + ] + ] + }, + "why": "Pattern 2 in the body, compiled and run twice: SERIALIZABLE is accepted as a table hint on a plain UPDATE, the insert branch fires only once, and the result is one row at n = 2. Learn documents HOLDLOCK as a synonym for SERIALIZABLE on the MERGE page, so both spellings appear in this skill and both need to be known good. Measured 2026-08-28. This is also the exact statement the two-shell check in Check it worked drives from 100 iterations." + }, + { + "sql": "CREATE TABLE dbo.probe_iv_sales (id INT NOT NULL PRIMARY KEY, region NVARCHAR(20) NOT NULL, amt INT NOT NULL);\nINSERT INTO dbo.probe_iv_sales (id, region, amt) VALUES (1, N'east', 10), (2, N'east', 20), (3, N'west', 5);\nGO\nCREATE VIEW dbo.probe_iv_totals WITH SCHEMABINDING AS\nSELECT region, SUM(amt) AS total, COUNT_BIG(*) AS n FROM dbo.probe_iv_sales GROUP BY region;\nGO\nCREATE UNIQUE CLUSTERED INDEX ux_probe_iv_totals ON dbo.probe_iv_totals (region);\nGO\nMERGE dbo.probe_iv_sales WITH (HOLDLOCK) AS t\nUSING (VALUES (2, N'east', 99)) AS s(id, region, amt) ON t.id = s.id\nWHEN MATCHED THEN UPDATE SET t.amt = s.amt\nWHEN NOT MATCHED BY SOURCE AND t.region = N'west' THEN DELETE;\nGO\nSELECT total, n FROM dbo.probe_iv_totals WITH (NOEXPAND) WHERE region = N'east';", + "expect": { + "rows": [ + [ + 109, + 2 + ] + ] + }, + "why": "The half of the indexed view claim that did not reproduce, asserted rather than assumed. A MERGE combining UPDATE with NOT MATCHED BY SOURCE THEN DELETE against a base table under an aggregated indexed view leaves the materialized rows agreeing with the recomputation: east becomes 10 + 99 = 109 over two rows once the west row is deleted. If a future engine build ever does break this, this probe goes red and the skill's wording has to change with it. Read through the view with NOEXPAND so the answer comes from the materialized rows and not from a recomputed plan." + }, + { + "sql": "CREATE TABLE dbo.probe_iv_check_sales (id INT NOT NULL PRIMARY KEY, region NVARCHAR(20) NOT NULL, amt INT NOT NULL);\nINSERT INTO dbo.probe_iv_check_sales (id, region, amt) VALUES (1, N'east', 10);\nGO\nCREATE VIEW dbo.probe_iv_check_totals WITH SCHEMABINDING AS\nSELECT region, SUM(amt) AS total, COUNT_BIG(*) AS n FROM dbo.probe_iv_check_sales GROUP BY region;\nGO\nCREATE UNIQUE CLUSTERED INDEX ux_probe_iv_check ON dbo.probe_iv_check_totals (region);\nGO\nDBCC CHECKTABLE ('dbo.probe_iv_check_totals') WITH EXTENDED_LOGICAL_CHECKS;", + "expect": { + "succeeds": true + }, + "why": "The Check it worked section tells a reader to run this, so the lane has to know the engine accepts it. Microsoft Learn states that on indexed views only physical consistency checks run by default and that EXTENDED_LOGICAL_CHECKS is what runs the logical ones, for Azure SQL Database. This asserts acceptance only, deliberately: what the check reports on a healthy view is not the claim, and asserting output text would tie the probe to a message format rather than to the option existing. Measured 2026-08-28." + } + ] + }, + "triggering": { + "explicit": [ + "Write me an upsert for this table on Azure SQL Database", + "Insert the row if it does not exist and update it if it does", + "Write a MERGE statement to sync this staging table into the target table", + "Is this MERGE safe to run while the application is writing to the same table", + "Port this ON CONFLICT DO UPDATE to Azure SQL Database", + "Review this upsert stored procedure before it goes to production" + ], + "implicit": [ + "we have two rows with the same key and nothing in the application inserts twice", + "this worked in testing and now we get violation of PRIMARY KEY constraint under load", + "the MERGE statement attempted to UPDATE or DELETE the same row more than once", + "my sync job deleted rows that were not in the batch", + "make this insert idempotent so retries do not create a second row", + "why is my upsert deadlocking now that we run it from several workers", + "should I wrap the check and the insert in a transaction", + "a MERGE statement must be terminated by a semi-colon" + ], + "routing": [ + "add or update these records from the incoming file", + "make this import re-runnable" + ], + "negative": [ + "Write me a paginated query for Azure SQL", + "How do I get the id of the row I just inserted", + "Parse this JSON column with OPENJSON", + "Should this document column be nvarchar or the json type", + "How long should the string columns in my index be", + "What collation should this database use", + "Why is this SELECT slow", + "Write a PostgreSQL upsert for my Postgres database" + ] + } +} diff --git a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql deleted file mode 120000 index e3c7b65..0000000 --- a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql +++ /dev/null @@ -1 +0,0 @@ -../../../skills/vector-search-azure-sql \ No newline at end of file diff --git a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/SKILL.md b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/SKILL.md new file mode 100644 index 0000000..fbe3051 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/SKILL.md @@ -0,0 +1,348 @@ +--- +name: vector-search-azure-sql +description: >- + Stores and searches vectors natively in Azure SQL Database: the vector type, VECTOR_DISTANCE, + the 1998 dimension ceiling, the DiskANN vector index, and the long list of places a vector + column is refused. Use when a schema needs an embedding column, when someone asks to "store + embeddings in SQL", "do similarity search", "cosine distance", "top k nearest neighbours", + "CREATE VECTOR INDEX", "VECTOR_SEARCH" or "WITH APPROXIMATE"; when a vector column is rejected + as a key, a constraint, a computed column or inside ORDER BY, GROUP BY, DISTINCT or UNION; and + when a similarity query returns the right rows but scans the whole table. This skill owns the + type and the query surface. The end to end pipeline is rag-on-azure-sql, generating embeddings + embeddings-and-external-models, and a vector column's place in a wider design + design-azure-sql-schema. +--- + +# Vector storage and search on Azure SQL Database + +The `vector` type behaves like no other column type in this engine, and the query that looks like +vector search is not the query that uses the vector index. + +Syntax and status re-checked 2026-09-03 against Microsoft Learn's `vector data type`, +`VECTOR_SEARCH` and `CREATE VECTOR INDEX` pages. Every error number and plan below was measured +2026-08-28 against a live Azure SQL Database (`SERVERPROPERTY('EngineEdition')` returns 5, General +Purpose serverless, compatibility level 170) and again against the local Azure SQL Database +container, which reports the same edition. They agreed on every error number below. + +## Build the fixture and the index first + +Do not read about the index. Build one. It is the only way to learn whether the preview has reached +your engine. + +```sql +CREATE TABLE dbo.chunks ( + id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, + body nvarchar(400) NOT NULL, + category nvarchar(40) NOT NULL, + embedding vector(5) NULL +); +GO +INSERT INTO dbo.chunks (body, category, embedding) +SELECT CONCAT(N'chunk ', value), N'general', + CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), + CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), + CAST(value * 0.05 AS float)) AS vector(5)) +FROM GENERATE_SERIES(1, 150); +GO +CREATE NONCLUSTERED INDEX ix_chunks_category ON dbo.chunks (category); +GO +DECLARE @t0 datetime2(3) = SYSUTCDATETIME(); +CREATE VECTOR INDEX vi_chunks ON dbo.chunks (embedding) + WITH (METRIC = 'cosine', TYPE = 'diskann'); +SELECT DATEDIFF(millisecond, @t0, SYSUTCDATETIME()) AS build_ms; +``` + +Measured over 140 rows: 145 ms on the local container, 276 ms in the cloud. If a plan, a document or +a model says this statement is not yet buildable, run it before believing it. You turn nothing +on first. Compatibility level is not the gate: column, insert, `VECTOR_DISTANCE` and approximate +search over an existing index all worked at level 150. `PREVIEW_FEATURES` is not the gate either: + +```sql +SELECT [name], [value] FROM sys.database_scoped_configurations WHERE [name] = 'PREVIEW_FEATURES'; +``` + +**Do not read a decision out of that number.** On the container it is not deterministic: six +freshly created databases on one engine read 0, 1, 0, 1, 1, 1 on 2026-09-03, while `model` and +`master` both read 0. Learn documents OFF as the default, scopes the requirement to the boxed +engine, and its own `CREATE VECTOR INDEX` example calls the setting "not needed for Azure SQL +Database". What matters is the statement above: the index builds and you set nothing. If a +document tells you to turn this on first, it is wrong whatever your database happens to +report. The type, `VECTOR_DISTANCE`, `VECTOR_NORM`, `VECTOR_NORMALIZE` and `VECTORPROPERTY` are +generally available; `CREATE VECTOR INDEX`, `VECTOR_SEARCH` and `TOP (N) WITH APPROXIMATE` are +preview and roll out by region, so a design that only performs with the index carries preview risk. + +## The correction: which query shape reaches the index + +An agent asked for fast similarity search writes this. It is wrong and nothing reports it: + +```sql +-- Correct results. Never uses a vector index. Computes a distance for EVERY row. +DECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]'; +SELECT TOP (10) id, body, VECTOR_DISTANCE('cosine', embedding, @q) AS distance +FROM dbo.chunks +ORDER BY VECTOR_DISTANCE('cosine', embedding, @q); +``` + +Measured plan with a current DiskANN index: `Clustered Index Scan`, then `Compute Scalar` +evaluating `vector_distance` per row, then `Sort`. The index is not in the plan. Learn states it +outright: vector distance is always exact and doesn't use any vector index, even if available. The +shape that does reach the index is a different statement, not a tuning option: + +```sql +-- Uses the vector index. Vector Index Seek in the measured plan. +DECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]'; +SELECT TOP (10) WITH APPROXIMATE t.id, t.body, r.distance +FROM VECTOR_SEARCH(TABLE = dbo.chunks AS t, COLUMN = embedding, + SIMILAR_TO = @q, METRIC = 'cosine') AS r +WHERE t.category = N'general' +ORDER BY r.distance; +``` + +Four shapes, measured on the same table and index: + +| Query | Measured plan | +|---|---| +| `ORDER BY VECTOR_DISTANCE(...)` with `TOP` | Clustered Index Scan, exact, full scan | +| `VECTOR_SEARCH` with `TOP` and no `WITH APPROXIMATE` | Clustered Index Scan, exact kNN | +| `VECTOR_SEARCH` with `TOP (N) WITH APPROXIMATE` | **Vector Index Seek**, approximate | +| `TOP (N) WITH APPROXIMATE` on a table with **no** vector index | Clustered Index Scan, **no error** | + +**Every wrong shape returns correct rows.** That is the whole hazard. A pipeline tested on a few +thousand rows passes, ships, then computes one distance per row per query forever, so the defect shows +up as latency and cost, not as a failure anyone can point at. Learn says a warning is +raised when no compatible index is found; nothing surfaced here. + +Turn the silent fallback into a failure. `WITH (FORCE_ANN_ONLY)` goes on the `VECTOR_SEARCH` alias +and needs an index and `WITH APPROXIMATE` present: + +```sql +DECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]'; +SELECT TOP (10) WITH APPROXIMATE t.id, r.distance +FROM VECTOR_SEARCH(TABLE = dbo.chunks AS t, COLUMN = embedding, + SIMILAR_TO = @q, METRIC = 'cosine') AS r WITH (FORCE_ANN_ONLY) +ORDER BY r.distance; +``` + +With no usable index that fails with `Msg 42227` instead of quietly scanning. Use it in a test, not +in production. + +### The older syntax an agent reaches for first + +`VECTOR_SEARCH(..., TOP_N = 10)` is what most published examples still show. Against a current index +it fails with `Msg 42274`. Match on that number, not the text: Learn prints "version 3 index" where +the engine measured here printed "newer index version". `TOP_N` belongs to an earlier index format +that cannot be upgraded in place, so move the count to `SELECT TOP (N) WITH APPROXIMATE`. + +`WITH APPROXIMATE` is also not a general modifier. Without a `VECTOR_SEARCH` it fails with +`Msg 42248`, and its `ORDER BY` is fixed: the `distance` column, ascending, nothing else, or +`Msg 42271`. Learn's own end to end example on the `CREATE VECTOR INDEX` page ends +`ORDER BY s.distance, t.title`, which its own `VECTOR_SEARCH` page says is `Msg 42271`. Copy the +rule, not the example. `GROUP BY`, window functions, `DISTINCT`, set operators, extra sort keys and +`CROSS APPLY` all need the approximate search in a subquery and the outer query doing the rest. + +## What VECTOR_DISTANCE returns, and the sign trap + +Three metrics exist and nothing else does: `'cosine'`, `'euclidean'`, `'dot'`. Anything else is +`Msg 42201`. The metric may be a variable and is case insensitive. + +```sql +DECLARE @q vector(3) = '[1,2,3]'; +SELECT v.label, + VECTOR_DISTANCE('cosine', CAST(v.txt AS vector(3)), @q) AS cosine, + VECTOR_DISTANCE('euclidean', CAST(v.txt AS vector(3)), @q) AS euclidean, + VECTOR_DISTANCE('dot', CAST(v.txt AS vector(3)), @q) AS dot +FROM (VALUES ('identical', '[1,2,3]'), ('near', '[1,2,4]'), ('far', '[4,5,6]')) AS v(label, txt); +``` + +| Row | `cosine` | `euclidean` | `dot` | +|---|---|---|---| +| `[1,2,3]`, identical | 5.96e-08 | 0 | **-14** | +| `[1,2,4]` | 0.0085 | 1 | -17 | +| `[4,5,6]` | 0.0254 | 5.196 | -32 | + +It returns a **distance**: smaller is closer, always, for all three. + +- **Cosine returns 1 minus the similarity**, on a range Learn gives as 0 to 2. An identical vector + scores near zero, so a threshold copied from a similarity based store inverts. +- **`dot` returns the negative inner product.** The identical row scores -14, not 14. A query that + sorts `dot` descending, or keeps the largest score, returns the **least** similar rows, with no + error and a full result set, and grounds its answer on the worst matches. +- Both arguments must share a base type. `float32` against `float16` is `Msg 42243`. + +## The type, in one place + +```sql +DECLARE @v vector(3) = '[1,2,3]'; +SELECT VECTORPROPERTY(@v, 'Dimensions') AS dims, -- 3 + VECTORPROPERTY(@v, 'BaseType') AS base_type, -- float32 + DATALENGTH(@v) AS bytes, -- 20 + CAST(@v AS nvarchar(max)) AS round_trip; +``` + +- **Dimensions are 1 to 1998**, which Learn states and the engine enforces: `vector(1999)` is + `Msg 2717`, `vector(0)` is `Msg 1001`. A populated `vector(1998)` has `DATALENGTH` 8000. + **The declared dimension cannot be changed later**, even on an empty table: `ALTER COLUMN` is + `Msg 42204`, so it is drop and recreate. A 3072 dimension model does not fit, and that fix + belongs at embedding time. +- **Base type is `float32`** unless `float16` is stated. Half precision is preview and is a surface + Learn does gate on `PREVIEW_FEATURES`, it crosses TDS as a JSON string rather than in binary, and + it cannot be compared with a `float32` vector. +- **It is written and read as a JSON array**, converting implicitly from `varchar`, `nvarchar` and + `json`. **The dimension in `CAST` must be a literal**: `CAST(@s AS vector(@d))` is `Msg 102`, so a + helper taking the dimension as a parameter needs dynamic SQL. +- **Clients that predate the protocol change see `nvarchar(max)`**, because Learn records that + `sp_describe_first_result_set` misreports the type, which is why an ORM maps it to a string. + Native handling needs `Microsoft.Data.SqlClient` 6.1.0 or the JDBC driver 13.1.0. + +## The restriction list + +No model infers this: a `vector` column looks like a column. Every row was measured. + +**Refused in a table definition** + +| Attempt | Error | +|---|---| +| `PRIMARY KEY` or `UNIQUE` on a vector column | `Msg 1919` | +| `FOREIGN KEY` referencing one | `Msg 1776`, no candidate key to reference | +| `CHECK` constraint | `Msg 1760` | +| `DEFAULT` constraint | `Msg 1752` | +| `PERSISTED` computed column over a vector function | `Msg 4936`, non-deterministic | +| Partition function or partition key | `Msg 7704` | +| Alias type with `CREATE TYPE` | `Msg 42212` | +| Assignment to `sql_variant` | `Msg 206` | +| Memory optimized table | Not supported | +| `ALTER COLUMN` to a different dimension | `Msg 42204` | + +**Allowed, though widely assumed otherwise:** `SPARSE`, a non-persisted +computed column, an `INCLUDE` column on a nonclustered index, a system versioned temporal table, a +table carrying a clustered columnstore index, `SELECT ... INTO`, `ALTER COLUMN` from `nvarchar` to +`vector`, `CASE`, `ISNULL`, `UNION ALL` and `FOR JSON`. Also parameters, return types and table +variables. + +**Refused in a query** + +| Attempt | Error | +|---|---| +| `ORDER BY` or `GROUP BY` the column itself | `Msg 42213`, cannot be compared or sorted | +| `SELECT DISTINCT` | `Msg 421` | +| `=`, `<`, `>`, `IN`, a join predicate, `MAX`, `MIN`, `COUNT(DISTINCT ...)` | `Msg 8117` | +| `UNION`, `INTERSECT`, `EXCEPT` | `Msg 5335`. `UNION ALL` is fine | +| `CREATE STATISTICS` on the column, or any b-tree or columnstore **key** | `Msg 1978` | +| `CAST` to `varbinary` | `Msg 529` | +| Inserting a JSON array of the wrong length | `Msg 42204` | +| Inserting a JSON array containing a string | `Msg 13670` | + +`IS NULL` and `IS NOT NULL` are the only comparisons that work. With no equality, +**deduplicating rows by their embedding is not a query you can write**. Deduplicate on a hash of the +source text, which Learn wants anyway: duplicates degrade recall. Always Encrypted is +unsupported, and `sp_verify_database_ledger` errors on a database holding a vector column. + +## What the index requires, and what it costs + +- **The table needs a clustered index.** Without one, `Msg 42254`. Learn says a primary key + clustered index; clustered primary keys on `int`, `bigint`, `nvarchar` and a composite all built + here, so `Msg 42217`'s narrower wording is stale, not the rule. Re-measured on a real logical + server 2026-09-08: `bigint` and a composite `(int, int)` both built, a heap was refused with + `Msg 42254` and not 42217, and **no input produced `Msg 42217` at all**. +- **At least 100 rows with a non-null vector must already exist**, or `Msg 42266`. A fresh database + has no index until it has content, and every query is silently exact until then. The fixture + inserts 150 for headroom. +- **One vector index per column.** A second, even with a different metric, is `Msg 42230`. The + metric is fixed at build time, and asking `VECTOR_SEARCH` for a different one **falls back + silently**. Measured 2026-09-06 against a `cosine` index on 150 rows: `METRIC = 'euclidean'` and + `METRIC = 'dot'` both returned a full ordered result set, no error and no warning, computed + exactly. This skill said `Msg 42227` here until that measurement, and the claim was backwards in + the one direction that matters, because a reader who believes the engine will stop them has no + reason to check. `Msg 42227` is raised only by `WITH (FORCE_ANN_ONLY)`, which is why that hint + belongs in the test that proves the index is being read. +- **The local Azure SQL Database container builds it too.** Open + `references/parity.md` in `rag-local-with-container` when a plan leans on the two engines + agreeing. + +| Operation | Result | +|---|---| +| `TRUNCATE TABLE` | `Msg 42232`. Drop the index, truncate, reload 100 rows, recreate | +| `ALTER INDEX ... REBUILD` or `DISABLE` | `Msg 42250`. There is no rebuild | +| `ALTER TABLE ... DROP COLUMN` on the indexed column | `Msg 5074` | +| Importing a dacpac or bacpac | Fails. The export is fine. Drop the indexes before export, recreate after the load | +| Partitioning the table | Not supported | + +This skill used to report that import row as unsettled and it is settled. Learn states it: the +import creates schema objects before it loads data, so the index is built against an empty table and +trips the 100 row minimum. +`sqlpackage-import-export` measured the round trip on 2026-08-31 and agreed with Learn. +Open it before planning either half, because it owns the sequence. + +`INSERT`, `UPDATE`, `DELETE` and `MERGE` are all supported on a current index and changes are +searchable after commit, which is new: earlier index versions made the table read only. Watch +`sys.dm_db_vector_indexes` for background maintenance, and **drop and recreate the index after +replacing most of the embeddings**: the graph was built for the old distribution and recall degrades +with no error. Filter predicates apply during the search, so a filtered top ten returns ten rows +when ten qualify. Give the filter columns a nonclustered index, as the fixture does. + +## Check it worked + +Three checks. The middle one settles what this skill exists for. + +```sql +-- 1. The index exists and is the current format. index_version must be 3. +SELECT i.name AS index_name, OBJECT_NAME(v.object_id) AS table_name, + JSON_VALUE(v.build_parameters, '$.Version') AS index_version +FROM sys.vector_indexes AS v +JOIN sys.indexes AS i ON v.object_id = i.object_id AND v.index_id = i.index_id; + +-- 2. The query reaches it. Read the plan and look for Vector Index Seek. +SET STATISTICS XML ON; +DECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]'; +SELECT TOP (10) WITH APPROXIMATE t.id, r.distance +FROM VECTOR_SEARCH(TABLE = dbo.chunks AS t, COLUMN = embedding, + SIMILAR_TO = @q, METRIC = 'cosine') AS r +ORDER BY r.distance; +SET STATISTICS XML OFF; + +-- 3. Ranking is the right way up: the nearest row has the SMALLEST distance. +DECLARE @p vector(5) = '[0.3,0.3,0.3,0.3,0.3]'; +SELECT MIN(VECTOR_DISTANCE('cosine', embedding, @p)) AS nearest, + MAX(VECTOR_DISTANCE('cosine', embedding, @p)) AS furthest +FROM dbo.chunks; +``` + +Run them non-interactively with the password in `SQLCMDPASSWORD`, keeping the output: + +```bash +sqlcmd -S .database.windows.net,1433 -d -U -C \ + -i check-vector-index.sql -o check-vector-index.out +``` + +Expected: check 1 returns one row per vector index with `index_version` `3`; an empty result set +means there is no index and every similarity query is scanning. Check 2's plan contains +`Vector Index Seek`; a plan holding only `Clustered Index Scan` and `Compute Scalar` is the defect, +whatever the rows look like. Check 3 returns `nearest` below `furthest`, both non-negative. +`sqlcmd` exiting 0 says nothing about the three, so read the output file. + +## Do not + +- Do not conclude the index is in use because results look right. Read the plan, or assert with + `FORCE_ANN_ONLY`. `WITH APPROXIMATE` is not a droppable hint: it changes the algorithm. +- Do not sort a `dot` distance descending, and do not present a cosine distance as a similarity. +- Do not declare `vector(3072)`. The ceiling is 1998, and the fix belongs at embedding time. +- Do not deduplicate on the embedding. There is no equality operator to do it with. +- Do not raise the compatibility level or set `PREVIEW_FEATURES = ON`. Neither is the gate, and the + container's `1` is not the cloud's value. +- Do not report that the index cannot be built on the local container. It builds there. +- Do not build the ingest, chunking and grounding pipeline here. That is `rag-on-azure-sql`. + +## References + +- [references/vector-restrictions-and-query-plans.md](references/vector-restrictions-and-query-plans.md) when a claim here + disagrees with what you are seeing, which on a preview surface happens: it holds the raw statements + behind every table above, the four plans and the reproduction steps. +- Read [vector data type](https://learn.microsoft.com/sql/t-sql/data-types/vector-data-type) before + choosing a dimension count or a driver, and + [CREATE VECTOR INDEX](https://learn.microsoft.com/sql/t-sql/statements/create-vector-index-transact-sql) + before creating an index in production, for preview status and the migration off the earlier + format. +- Read [VECTOR_SEARCH](https://learn.microsoft.com/sql/t-sql/functions/vector-search-transact-sql) + when a query will not compile alongside `WITH APPROXIMATE`, for the subquery patterns, and + [Feature availability by region](https://learn.microsoft.com/azure/azure-sql/database/region-availability#vector-search) + before promising the preview in a region you have not built one in. diff --git a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/references/vector-restrictions-and-query-plans.md b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/references/vector-restrictions-and-query-plans.md new file mode 100644 index 0000000..484c05a --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/references/vector-restrictions-and-query-plans.md @@ -0,0 +1,272 @@ +# Vector restrictions and query plans, measured 2026-08-28 + +## Contents + +- [How this was run](#how-this-was-run) +- [The four query shapes and their plans](#the-four-query-shapes-and-their-plans) +- [Metric semantics](#metric-semantics) +- [Type and dimension limits](#type-and-dimension-limits) +- [Table definition: refused](#table-definition-refused) +- [Table definition: allowed](#table-definition-allowed) +- [Query surface: refused](#query-surface-refused) +- [Vector index: requirements](#vector-index-requirements) +- [Vector index: what it blocks](#vector-index-what-it-blocks) +- [Approximate search syntax rules](#approximate-search-syntax-rules) +- [Claims that did not hold](#claims-that-did-not-hold) +- [Reproducing this](#reproducing-this) + +## How this was run + +Every statement was executed twice: once against an Azure SQL Database provisioned for the run +(General Purpose serverless, two vCores, compatibility level 170, Microsoft Entra authentication +only) and once against the local Azure SQL Database container. Both engines reported +`SERVERPROPERTY('EngineEdition') = 5`. **Every error number below was identical on both**, which is +the reason each row is stated as an engine behaviour rather than as a service behaviour. The one +reading that cannot be stated at all is `PREVIEW_FEATURES`, item 8 under Claims that did not hold. + +Nothing here was taken from documentation. Where the documentation and the engine disagreed, the +disagreement is recorded under [Claims that did not hold](#claims-that-did-not-hold). + +## The four query shapes and their plans + +Table: 20000 rows, `vector(64)` column, one DiskANN vector index with `METRIC = 'cosine'`, index +format version 3. Plans captured with `SET STATISTICS XML ON`. + +| Query | Operators in the actual plan | +|---|---| +| `SELECT TOP (5) id ... ORDER BY VECTOR_DISTANCE('cosine', e, @q)` | `Sort`, `Compute Scalar` defining `vector_distance`, `Clustered Index Scan` | +| `SELECT TOP (5) ... FROM VECTOR_SEARCH(...) ORDER BY s.distance` | `Top`, `Sort`, `Compute Scalar`, `Filter`, `Clustered Index Scan` | +| `SELECT TOP (5) WITH APPROXIMATE ... FROM VECTOR_SEARCH(...) ORDER BY s.distance` | `Top`, `Nested Loops`, **`Vector Index Seek`** on the vector index, `Clustered Index Seek` on the primary key | +| `SELECT TOP (2) WITH APPROXIMATE ... FROM VECTOR_SEARCH(...)` against a table with **no** vector index | `Sort`, `Compute Scalar`, `Filter`, `Clustered Index Scan`. No error and no warning surfaced to the client | + +`WITH (FORCE_ANN_ONLY)` on the `VECTOR_SEARCH` alias succeeded on the indexed table and failed on +the unindexed one with `Msg 42227`. That is the only cheap way to turn the silent fallback into a +failure inside a test. + +## Metric semantics + +`DECLARE @q vector(3) = '[1,2,3]';` against rows `[1,2,3]`, `[4,5,6]`, `[1,2,4]`. + +| Row | `cosine` | `euclidean` | `dot` | +|---|---|---|---| +| `[1,2,3]` | 5.960464477539063e-08 | 0 | -14 | +| `[4,5,6]` | 0.025368213653564453 | 5.196152210235596 | -32 | +| `[1,2,4]` | 0.008539915084838867 | 1 | -17 | + +- `'manhattan'` and `'negative dot product'`: `Msg 42201, The requested distance metric '' is + not supported by vector_distance.` +- `'COSINE'` in upper case: accepted. +- The metric supplied as a `varchar` variable rather than a literal: accepted. +- Dimension mismatch between the two arguments: `Msg 42204, The vector dimensions 3 and 4 do not + match.` +- `float16` against `float32`: `Msg 42243, VECTOR_DISTANCE function does not support different base + types for vector arguments.` + +## Type and dimension limits + +| Statement | Result | +|---|---| +| `CREATE TABLE t (v vector(1))` | Succeeded | +| `CREATE TABLE t (v vector(0))` | `Msg 1001, Length or precision specification 0 is invalid` | +| `CREATE TABLE t (v vector(1998))` | Succeeded | +| `CREATE TABLE t (v vector(1999))` | `Msg 2717, The size (1999) given to the column 'v' exceeds the maximum allowed (1998)` | +| `DATALENGTH` of a populated `vector(1998)` | 8000 bytes | +| `sys.columns.max_length` for a `vector(1998)` | 8000 | +| `SELECT CAST(v AS nvarchar(max))` | Succeeded, returned a JSON array such as `[1.0000000e+000,2.0000000e+000,3.0000000e+000]` | +| `SELECT CAST(v AS varbinary(max))` | `Msg 529, Explicit conversion from data type vector to varbinary(max) is not allowed` | +| `CAST('[1,2,3]' AS vector(@d))` with `@d` a variable | `Msg 102, Incorrect syntax near '@d'` | +| Insert `'[1,2]'` into a `vector(3)` | `Msg 42204` | +| Insert `'[1,"a",3]'` into a `vector(3)` | `Msg 13670, Input JSON is not a valid Vector : 'String not Supported'` | +| `VECTORPROPERTY(v, 'Dimensions')` | Returned the dimension count | +| `VECTORPROPERTY(v, 'BaseType')` | Returned `float32`, or `float16` where declared | +| `VECTOR_NORM(v, 'norm2')` and `'norm1'` | Both succeeded | +| `VECTOR_NORMALIZE(v, 'norm2')` | Succeeded, returned a vector | +| `JSON_ARRAY_TO_VECTOR`, `VECTOR_TO_JSON_ARRAY` | `Msg 195`, not recognised. These names do not exist | +| Column, insert and `VECTOR_DISTANCE` at compatibility level 150 | All succeeded | +| Approximate search over an existing index at compatibility level 150 | Succeeded | +| `vector(3, float16)` on the cloud database | The `CREATE TABLE` succeeded | + +## Table definition: refused + +| Statement | Error | +|---|---| +| `v vector(3) PRIMARY KEY` | `Msg 1919`, invalid for use as a key column in an index, then `Msg 1750` | +| `v vector(3) UNIQUE` | `Msg 1919`, then `Msg 1750` | +| `CREATE CLUSTERED INDEX ix ON t(v)` | `Msg 1978`, invalid for use as a key column in an index or statistics | +| `CREATE INDEX ix ON t(v)` | `Msg 1978` | +| `v vector(3) REFERENCES other(v)` | `Msg 1776`, no candidate key in the referenced table | +| `v vector(3) CHECK (...)` | `Msg 1760, Constraints of type CHECK cannot be created on columns of type vector` | +| `v vector(3) DEFAULT '[0,0,0]'` | `Msg 1752`, invalid for creating a default constraint | +| `c AS VECTOR_NORM(v,'norm2') PERSISTED` | `Msg 4936`, cannot be persisted because the column is non-deterministic | +| `CREATE PARTITION FUNCTION pf (vector(3)) ...` | `Msg 7704, The type 'sys.vector' is not valid for this operation` | +| `CREATE TYPE myvec FROM vector(3)` | `Msg 42212, Cannot create alias types from a vector datatype` | +| `SET @sv = CAST('[1,2,3]' AS vector(3))` where `@sv` is `sql_variant` | `Msg 206, Operand type clash: vector is incompatible with sql_variant` | +| `CREATE STATISTICS s ON t(v)` | `Msg 1978` | +| `ALTER TABLE t ALTER COLUMN v vector(4)` from `vector(3)`, table empty | `Msg 42204, The vector dimensions 3 and 4 do not match` | +| Memory optimized table containing a vector column | Not reachable on the tier tested, which reported the tier restriction rather than a type restriction. Documented as unsupported | + +## Table definition: allowed + +Each of these was created and then confirmed in the catalogue views rather than assumed from the +absence of an error. + +| Statement | Confirmed by | +|---|---| +| `v vector(3) NULL` and `NOT NULL` | Insert of `NULL` and of a value | +| `v vector(3) SPARSE NULL` | `sys.columns.is_sparse = 1` on the vector column | +| `c AS VECTOR_NORM(v,'norm2')`, not persisted | Created | +| `c AS v`, a computed column of vector type | Created | +| `CREATE INDEX ix ON t(id) INCLUDE (v)` | `sys.index_columns.is_included_column = 1` for the vector column | +| `CREATE CLUSTERED COLUMNSTORE INDEX cci ON t` where `t` has a vector column | `sys.indexes.type_desc = CLUSTERED COLUMNSTORE` | +| System versioned temporal table with a vector column | `sys.tables.temporal_type_desc = SYSTEM_VERSIONED_TEMPORAL_TABLE` | +| `DECLARE @t TABLE (v vector(3))` | Insert and count | +| `CREATE TYPE vt AS TABLE (v vector(3))` | Created. This is a table type, unlike the alias type above | +| Stored procedure parameter, scalar function return type, inline table function parameter | All created | +| `SELECT id, e INTO t2 FROM t` | Created | +| `ALTER TABLE t ALTER COLUMN x vector(3)` from `nvarchar(100)` | Succeeded | +| A view selecting a vector column | Created | +| `ALTER TABLE t ENABLE CHANGE_TRACKING` | Succeeded | +| `FOR JSON PATH` over a vector column | Returned the vector as a JSON array | +| Two vector columns in one table, including a `vector(1998)` | Created | +| Row level security filter predicate on a table with a vector column | Enforced, and enforced through approximate search as well | + +## Query surface: refused + +| Statement | Error | +|---|---| +| `ORDER BY e` | `Msg 42213, The vector data types cannot be compared or sorted, except when using the IS NULL operator` | +| `GROUP BY e` | `Msg 42213` | +| `SELECT DISTINCT e` | `Msg 421, The vector data type cannot be selected as DISTINCT because it is not comparable` | +| `WHERE e = '[1,2,3]'`, `a.e = b.e`, `e IN (...)` | `Msg 8117, Operand data type vector is invalid for equal to operator` | +| `WHERE e > '[1,2,3]'` | `Msg 8117`, greater than operator | +| `MAX(e)` | `Msg 8117`, max operator | +| `COUNT(DISTINCT e)` | `Msg 8117`, count operator | +| `SELECT e ... UNION SELECT e ...` | `Msg 5335, The data type vector cannot be used as an operand to the UNION, INTERSECT or EXCEPT operators because it is not comparable` | + +Working: `IS NULL`, `IS NOT NULL`, `UNION ALL`, `CASE`, `ISNULL`, and `ORDER BY +VECTOR_DISTANCE(...)`. + +`sp_describe_first_result_set` over a query selecting a vector column reported system type 231, +`nvarchar(max)`. That is why a client without current driver support sees a string. + +## Vector index: requirements + +| Statement | Result | +|---|---| +| `CREATE VECTOR INDEX` on a table with 0 rows | `Msg 42266, Cannot create a vector index. The table contains only 0 rows with non-null vectors, but at least 100 are required for vector index creation` | +| The same with 50 rows | `Msg 42266`, quoting 50 | +| The same with 500 rows | Succeeded. `sys.vector_indexes` reported `DiskANN`, `COSINE`, `build_parameters` `{"StartId":"...", "L":"48", "R":"48", "Version":"3"}` | +| On a heap with no clustered index | `Msg 42254, Clustered index is required on table '
' to create a vector index` | +| On a table whose primary key is nonclustered | `Msg 42254` | +| On a clustered primary key of type `int` | Succeeded | +| On a clustered primary key of type `bigint` | Succeeded | +| On a clustered primary key of type `nvarchar(20)` | Succeeded | +| On a composite clustered primary key of two `int` columns | Succeeded | +| `WITH (METRIC='cosine')`, `TYPE` omitted | Succeeded, defaulted to DiskANN | +| `WITH (METRIC='dot', TYPE='diskann')` | Succeeded | +| A second vector index on the same column with a different metric | `Msg 42230, Cannot create vector index on column '' because it already has an existing vector index` | +| On a `vector(1998)` column with 150 rows | Succeeded | +| `VECTOR_SEARCH` naming a metric with no matching index | `Msg 42227, Cannot find a vector index with metric '' on column ''` | + +## Vector index: what it blocks + +| Statement | Result | +|---|---| +| `TRUNCATE TABLE` | `Msg 42232, TRUNCATE TABLE statement failed because table '
' has a vector index on it` | +| `ALTER INDEX ... REBUILD` | `Msg 42250, One or more of the specified ALTER INDEX options is unsupported for a Vector Index` | +| `ALTER INDEX ... DISABLE` | `Msg 42250` | +| `ALTER TABLE ... DROP COLUMN` on the indexed column | `Msg 5074`, then `Msg 4922` | +| `INSERT`, `UPDATE`, `DELETE` | All succeeded, and the changed rows were searchable | + +## Approximate search syntax rules + +| Statement | Result | +|---|---| +| `VECTOR_SEARCH(..., TOP_N = 3)` against a version 3 index | `Msg 42274, Vector search with newer index version does not support explicit TOP_N parameter` | +| `TOP_N = 10000` | `Msg 42253`, TOP_N and L must both be between 1 and 2000 | +| `SELECT TOP (N) WITH APPROXIMATE` with no `VECTOR_SEARCH` in the query | `Msg 42248, APPROXIMATE cannot be used in a query without VECTOR_SEARCH` | +| `WITH APPROXIMATE` with no `ORDER BY` | `Msg 42248, APPROXIMATE cannot be used in a query without ORDER BY` | +| `ORDER BY r.distance DESC` | `Msg 42271, TOP WITH APPROXIMATE and VECTOR_SEARCH requires ORDER BY on distance column ascending, and no other columns` | +| `ORDER BY r.distance, t.id` | `Msg 42271` | +| An aggregate in the same select list | `Msg 42271` | +| `WHERE` predicate alongside `WITH APPROXIMATE` | Succeeded, and the filter was applied during the search | +| `INNER JOIN` to another table alongside `WITH APPROXIMATE` | Succeeded | + +## Claims that did not hold + +1. **The catalog note said "type GA, index preview", and both halves held.** The addition is that + `VECTOR_SEARCH` and `TOP (N) WITH APPROXIMATE` are part of the same preview as the index, not + part of the generally available type, and that the type works at compatibility level 150 rather + than needing 170. +2. **`Msg 42217` states that a table "must have a clustered primary key on a single 4 byte INT + column to create a vector index".** Measured, a clustered primary key on `bigint`, on + `nvarchar(20)` and on a two column composite all accepted a vector index. The message appears to + be narrower than the behaviour, and the rule that actually held was `Msg 42254`, a clustered + index is required. +3. **The `Msg 42274` text differs from the published text.** The engine returned "Vector search + with newer index version does not support explicit TOP_N parameter". The documentation quotes + "Vector search with version 3 index does not support explicit TOP_N parameter". Match on the + number, not on the words. +4. **Widely repeated claim that a vector column cannot appear in an index at all.** It cannot be a + key column, and it **can** be an `INCLUDE` column. Confirmed in `sys.index_columns`. +5. **Widely repeated claim that a vector index makes the table read only.** That was true of an + earlier index format. On a version 3 index every DML statement succeeded and the change was + searchable. +6. The dynamic management view column naming the outstanding maintenance differed between the two + engines tested. Read the column list from the view rather than hard coding a name. +7. **Widely repeated claim that a DiskANN vector index cannot be built on the local Azure SQL + Database container.** It builds. The parity run that settles it is + `rag-local-with-container`'s, which built the same index over the same 140 rows on both + engines, 145 ms on the container against 276 ms in the cloud, with approximate search returning + the same rows from both. That evidence is recorded there and is not copied here. +8. **Widely repeated claim that `PREVIEW_FEATURES = ON` is a prerequisite for creating a vector + index.** It is not, and **the setting's value on the container is not deterministic, so this + file asserts no value for it.** + + Measured 2026-09-03 on a single container: six freshly created databases returned `0`, `1`, + `0`, `1`, `1`, `1`, while `model` and `master` both returned `0`. The variation does not track + engine uptime cleanly. Two earlier revisions of this file each recorded a value, first `0` and + then `1`, and **each was one sample of something that varies.** A probe built on either would + have gone red about half the time on a claim that was never the point. + + Microsoft Learn documents `OFF` as the default, scopes the requirement to the boxed engine, and + calls the setting not needed for Azure SQL Database. + + The claim that survives is the one that matters and it is unaffected: nobody runs `ALTER + DATABASE SCOPED CONFIGURATION`. With 150 rows carrying a `vector(4)` value, + `CREATE VECTOR INDEX ... WITH (METRIC = 'cosine', TYPE = 'diskann')` completed in 74 ms, listed + in `sys.vector_indexes`, and served a `TOP (3) WITH APPROXIMATE` search. + +## Reproducing this + +Provision a database, or start the local container, then: + +```sql +CREATE TABLE dbo.chunks (id INT IDENTITY PRIMARY KEY, body NVARCHAR(400), embedding VECTOR(4)); + +DECLARE @i INT = 1; +WHILE @i <= 500 +BEGIN + INSERT dbo.chunks (body, embedding) + VALUES (CONCAT('sample chunk ', @i), + CAST(CONCAT('[', RAND(CHECKSUM(NEWID())), ',', RAND(CHECKSUM(NEWID())), ',', + RAND(CHECKSUM(NEWID())), ',', RAND(CHECKSUM(NEWID())), ']') AS VECTOR(4))); + SET @i += 1; +END; + +CREATE VECTOR INDEX vi_chunks ON dbo.chunks (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann'); + +SET STATISTICS XML ON; +DECLARE @q VECTOR(4) = '[0.1,0.2,0.3,0.4]'; + +-- exact, scans +SELECT TOP (5) id FROM dbo.chunks ORDER BY VECTOR_DISTANCE('cosine', embedding, @q); + +-- approximate, seeks the vector index +SELECT TOP (5) WITH APPROXIMATE t.id, r.distance +FROM VECTOR_SEARCH(TABLE = dbo.chunks AS t, COLUMN = embedding, SIMILAR_TO = @q, METRIC = 'cosine') AS r +ORDER BY r.distance; +``` + +Read the two plans. One contains `Vector Index Seek` and one does not. Everything else in this file +is a single statement run the same way. diff --git a/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/skill.spec.jsonc b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/skill.spec.jsonc new file mode 100644 index 0000000..357e813 --- /dev/null +++ b/plugins/microsoft-sql-vscode/skills/vector-search-azure-sql/skill.spec.jsonc @@ -0,0 +1,404 @@ +{ + "$schema": "../../catalog/skill.spec.schema.json", + "id": "vector-search-azure-sql", + "domain": "ai-vector-and-rag", + "value": ["not-in-training-data", "corrects-a-wrong-assumption", "changes-too-fast-to-memorize"], + "correction": "Agents write similarity search on Azure SQL Database as SELECT TOP (k) ... ORDER BY VECTOR_DISTANCE('cosine', embedding, @q) and believe a vector index accelerates it. Measured on 2026-08-28 against a live Azure SQL Database carrying a current DiskANN index, and reproduced identically on the local Azure SQL Database container, that query compiles to a Clustered Index Scan with a per row Compute Scalar and never touches the index; only SELECT TOP (N) WITH APPROXIMATE over VECTOR_SEARCH ordered by the distance column produces a Vector Index Seek. Every wrong shape returns the correct rows and raises nothing, including WITH APPROXIMATE against a table that has no vector index at all, so the mistake passes every test at development scale and appears in production only as latency and compute burn on every single query, on a table that will keep growing. The older TOP_N parameter that most published examples still show now fails outright with Msg 42274. The second half of the correction is that a vector column is not an ordinary column: it cannot be a key, a CHECK or DEFAULT constraint, a statistics target, a partition key or a persisted computed column, it cannot be compared, sorted, grouped, deduplicated, aggregated or carried across a UNION, its dimension cannot be changed by ALTER COLUMN even on an empty table, the ceiling is 1998 dimensions so a 3072 dimension embedding model does not fit, and the index it needs refuses to build under 100 rows and then blocks TRUNCATE TABLE, ALTER INDEX and data-tier package import for the life of the table. Two further beliefs a model states unprompted are wrong and are corrected here. It says a DiskANN vector index cannot be built on the local Azure SQL Database container; the same index over the same 140 rows built on the container in 145 ms and in the cloud in 276 ms, and approximate search returned the same rows from both, measured in rag-local-with-container's parity reference. It also says PREVIEW_FEATURES = ON is a prerequisite for creating a vector index; it is not. The index builds without it and nobody runs ALTER DATABASE SCOPED CONFIGURATION, which is the whole claim. The setting's value on the container is not worth reading and not worth asserting: measured 2026-09-03, six freshly created databases on one container read 0, 1, 0, 1, 1, 1, while model and master both read 0. Microsoft Learn documents OFF as the default, scopes the requirement to the boxed engine, and its own CREATE VECTOR INDEX example calls the setting not needed for Azure SQL Database.", + "value_probe": { + "gate": "The answer identifies the shape of the query itself as the reason the index is never read, and gives a retrieval form that does read it, instead of proposing statistics, service tier, hardware or an index rebuild as the remedy.", + "task": "Our product catalog on Azure SQL Database has about 4 million rows in dbo.products, each carrying a 1536 dimension embedding in a column called embedding. We followed a tutorial and created the index that is supposed to make these lookups fast, and it built without complaining. Search still takes several seconds, and the elapsed time creeps up every week as rows are added. This is what the API runs:\n\n```sql\nSELECT TOP (10) product_id, name\nFROM dbo.products\nORDER BY VECTOR_DISTANCE('cosine', embedding, @query);\n```\n\nTwo other things are on the plan for next quarter and I want to know now if they are going to work. We want to move to a larger embedding model that emits 3072 numbers per row, and we want to stop storing duplicate rows by putting a UNIQUE constraint across (tenant_id, embedding). Tell us why the search is still slow and whether those two plans hold up.", + "states": [ + "The query as written never reads the index and scans every row, so the index it built is doing nothing for this statement.", + "The wrong shape returns correct rows and raises no error, so the problem is invisible in testing and shows up only as latency and compute cost.", + "A vector column has a hard ceiling on its declared dimension that a 3072 number model exceeds, so that model cannot be stored in one such column.", + "A vector column cannot be part of a unique constraint, a primary key or any index key, so deduplicating on it is not available.", + "The declared dimension of an existing vector column cannot be changed by ALTER COLUMN, even when the table holds no rows." + ], + "refutes": [ + "The query as written does use the vector index, so the remaining latency is a statistics, hardware, service tier or index maintenance problem.", + "Turning PREVIEW_FEATURES on with ALTER DATABASE SCOPED CONFIGURATION is a prerequisite before a vector index can be created on Azure SQL Database.", + "A vector column can participate in a unique constraint or a primary key, so constraining the embedding column will deduplicate the rows.", + "An embedding of 3072 dimensions fits in an Azure SQL Database vector column, so the model swap needs nothing beyond widening the declared dimension.", + "The dimension of an existing vector column can be altered in place as long as the table is empty." + ] + }, + "posture": ["read", "write", "execute"], + "maturity": "preview", + "applies_to": ["azure-sql-db", "azure-sql-db-container"], + "related": [ + "rag-on-azure-sql", + "embeddings-and-external-models", + "rag-local-with-container", + "design-azure-sql-schema", + "vector-index-diskann", + "vector-index-staleness", + "diagnose-vector-search-latency", + "diagnose-vector-recall-drop", + "full-text-and-hybrid-ranking", + "azuresql-db-rag" + ], + "validation": { + "target": "both", + "setup": [ + "a database reachable with permission to create a table and an index", + "a table with a clustered primary key, a vector column and at least 100 rows carrying a non-null vector" + ], + "assert": [ + "the actual execution plan of every similarity query expected to use the index contains a Vector Index Seek", + "no query text contains the TOP_N parameter", + "any query left as ORDER BY VECTOR_DISTANCE carries a comment stating that an exact scan is intended", + "results are ordered by distance ascending, and no dot product result is treated as a similarity score", + "every declared dimension is between 1 and 1998 and equals what the embedding model is asked to produce", + "no vector column appears as a key, a CHECK or DEFAULT constraint, a persisted computed column, a partition key or a statistics target", + "deduplication keys off the source text or a hash of it rather than off the embedding", + "the vector index is created after the data load, and is dropped before a bacpac export and recreated after the import", + "any process that truncates or bulk replaces the table drops and recreates the vector index", + "no plan makes PREVIEW_FEATURES a precondition for a vector column or a vector index, none reads the container value of that setting as the cloud's, and none defers the index because the target engine is the local container" + ], + "probes": [ + { + "sql": "CREATE TABLE dbo.probe_vs_build (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, category nvarchar(20) NOT NULL, embedding vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_build (category, embedding)\nSELECT N'general', CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nDECLARE @t0 datetime2(3) = SYSUTCDATETIME();\nCREATE VECTOR INDEX vi_probe_vs_build ON dbo.probe_vs_build (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');\nDECLARE @ms int = DATEDIFF(millisecond, @t0, SYSUTCDATETIME());\nSELECT JSON_VALUE(v.build_parameters, '$.Version') AS index_version,\n CASE WHEN @ms < 30000 THEN 1 ELSE 0 END AS built_within_30s\nFROM sys.vector_indexes AS v\nJOIN sys.indexes AS i ON v.object_id = i.object_id AND v.index_id = i.index_id\nWHERE i.name = 'vi_probe_vs_build';", + "expect": { + "rows": [ + [ + "3", + 1 + ] + ] + }, + "why": "The headline claim, made executable: CREATE VECTOR INDEX is not theoretical on this engine, it builds. Measured 2026-08-28 at 145 ms on the container and 276 ms in the cloud over 140 rows, so 30 seconds is slack, not a threshold. The version column asserts the current DiskANN format, which Microsoft Learn's migration section says must read 3 and which is what makes the TOP_N probe below fail the way it does. An empty result set here means the index did not build at all." + }, + { + "sql": "SELECT COUNT(*) AS setting_exists\nFROM sys.database_scoped_configurations\nWHERE name = 'PREVIEW_FEATURES';", + "expect": { + "rows": [ + [ + 1 + ] + ] + }, + "why": "This asserts that the setting EXISTS and deliberately asserts nothing about its value, because its value is not deterministic. Measured 2026-09-03 on one container: six freshly created databases read 0, 1, 0, 1, 1, 1, while model and master both read 0, and the variation does not track engine uptime cleanly. Two earlier revisions of this probe each asserted a value, first 0 and then 1, and each was a single sample of something that varies, so the lane would have gone red roughly half the time on a claim that was never the point. What the skill actually needs is the probe above: the vector index builds, and nobody runs ALTER DATABASE SCOPED CONFIGURATION to make it build. Microsoft Learn documents OFF as the default, scopes the requirement to the boxed engine, and calls the setting not needed for Azure SQL Database. Do not reintroduce a value assertion here without many samples across many fresh containers." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_min (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, embedding vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_min (embedding)\nSELECT CAST(JSON_ARRAY(CAST(value AS float), 1.0, 2.0) AS vector(3)) FROM GENERATE_SERIES(1, 99);\nGO\nCREATE VECTOR INDEX vi_probe_vs_min ON dbo.probe_vs_min (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42266 + }, + "why": "99 rows is one short of the minimum, and the refusal is why a deployment that creates the index before loading data cannot work. Microsoft Learn states both the 100 row minimum and Msg 42266 by number on the CREATE VECTOR INDEX page." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_topn (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, category nvarchar(20) NOT NULL, embedding vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_topn (category, embedding)\nSELECT N'general', CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX vi_probe_vs_topn ON dbo.probe_vs_topn (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(5) = '[0.3,0.3,0.3,0.3,0.3]';\nSELECT TOP (3) t.id, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_vs_topn AS t, COLUMN = embedding, SIMILAR_TO = @q, METRIC = 'cosine', TOP_N = 3) AS r\nORDER BY r.distance;", + "expect": { + "error": 42274 + }, + "why": "The syntax most published examples still show, refused by a current index. Microsoft Learn gives the number on both the VECTOR_SEARCH and CREATE VECTOR INDEX pages. The probe asserts the number only, because Learn prints \"version 3 index\" where the engine measured on 2026-08-28 printed \"newer index version\". If this ever passes silently the index built above is not the current format and the rest of the skill's query guidance needs re-reading." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_trunc (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, category nvarchar(20) NOT NULL, embedding vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_trunc (category, embedding)\nSELECT N'general', CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX vi_probe_vs_trunc ON dbo.probe_vs_trunc (embedding) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nTRUNCATE TABLE dbo.probe_vs_trunc;", + "expect": { + "error": 42232 + }, + "why": "What the index takes away, and the one that breaks reload pipelines written before the index existed. Microsoft Learn states TRUNCATE TABLE is not allowed on a vector-indexed table without giving a number; Msg 42232 is this skill's own measurement of 2026-08-28." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_noindex (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, embedding vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_noindex (embedding) VALUES ('[1,2,3]'), ('[1,2,4]'), ('[4,5,6]'), ('[0,0,1]'), ('[9,9,9]');\nGO\nDECLARE @q vector(3) = '[1,2,3]';\nSELECT id FROM (\n SELECT TOP (2) WITH APPROXIMATE t.id, r.distance\n FROM VECTOR_SEARCH(TABLE = dbo.probe_vs_noindex AS t, COLUMN = embedding, SIMILAR_TO = @q, METRIC = 'cosine') AS r\n ORDER BY r.distance\n) AS nearest\nORDER BY id;", + "expect": { + "rows": [ + [ + 1 + ], + [ + 2 + ] + ] + }, + "why": "The silent fallback, measured rather than described. There is no vector index on this table and WITH APPROXIMATE still returns correct rows with no error and no client-visible warning, which is exactly why the wrong query shape survives every test at development scale. It asserts the two nearest ids rather than their float text, so the probe measures the behaviour and not sqlcmd's number formatting." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_forceann (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, embedding vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_forceann (embedding) VALUES ('[1,2,3]'), ('[1,2,4]'), ('[4,5,6]'), ('[0,0,1]'), ('[9,9,9]');\nGO\nDECLARE @q vector(3) = '[1,2,3]';\nSELECT TOP (2) WITH APPROXIMATE t.id, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_vs_forceann AS t, COLUMN = embedding, SIMILAR_TO = @q, METRIC = 'cosine') AS r WITH (FORCE_ANN_ONLY)\nORDER BY r.distance;", + "expect": { + "error": 42227 + }, + "why": "The same statement as the probe above with one table hint added, and the reason the skill recommends that hint in tests: it converts the silent scan into a failure. Microsoft Learn states FORCE_ANN_ONLY requires a vector index and that the query fails without one; Msg 42227 is this skill's measurement of 2026-08-28." + }, + { + "sql": "DECLARE @q vector(3) = '[1,2,3]';\nSELECT CASE WHEN VECTOR_DISTANCE('dot', @q, @q) < 0 THEN 1 ELSE 0 END AS dot_of_identical_is_negative,\n CASE WHEN VECTOR_DISTANCE('cosine', @q, @q) < 0.001 THEN 1 ELSE 0 END AS cosine_of_identical_is_near_zero;", + "expect": { + "rows": [ + [ + 1, + 1 + ] + ] + }, + "why": "The sign trap in two comparisons. A vector against itself scores negative under dot and near zero under cosine, so code that keeps the largest score or reads cosine as a similarity ranks the worst matches first with no error. Microsoft Learn documents dot as the negative dot product and cosine on a 0 to 2 distance range." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_dims (id int NOT NULL PRIMARY KEY, v vector(1999) NOT NULL);", + "expect": { + "error": 2717 + }, + "why": "The ceiling that decides which embedding model fits. Microsoft Learn states the maximum is 1998 dimensions; Msg 2717, the size given to the column exceeds the maximum allowed, is this skill's measurement of the refusal at 1999. This is the check that catches a 3072 dimension model being designed in." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_eq (id int NOT NULL PRIMARY KEY, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_eq (id, v) VALUES (1, '[1,2,3]'), (2, '[1,2,3]');\nSELECT COUNT(*) AS duplicates FROM dbo.probe_vs_eq WHERE v = CAST('[1,2,3]' AS vector(3));", + "expect": { + "error": 8117 + }, + "why": "Two rows carrying the identical embedding, and no way to write the query that finds them. Microsoft Learn states vectors do not support comparison operators; Msg 8117 is this skill's measurement. This is why deduplication has to key off a hash of the source text." + } + , + { + "sql": "CREATE TABLE dbo.probe_vs_dim_over (id int, v vector(1999));", + "expect": { + "error": 2717 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The skill states the ceiling is 1998 dimensions, which is why a 3072 number model does not fit in one column. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_dim_zero (id int, v vector(0));", + "expect": { + "error": 1001 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The other end of the declared range the skill gives as 1 to 1998. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "DECLARE @s nvarchar(100) = N'[1,2,3]';\nDECLARE @d int = 3;\nSELECT CAST(@s AS vector(@d)) AS v;", + "expect": { + "error": 102 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The skill says the dimension in CAST must be a literal, so a helper taking it as a parameter needs dynamic SQL. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_unique (id int NOT NULL, v vector(3) NOT NULL, CONSTRAINT uq_probe_vs UNIQUE (v));", + "expect": { + "error": 1919 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The restriction list's first row, and the half of the correction that says deduplicating on an embedding is not a query you can write. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_fk_p (v vector(3) NOT NULL);\nCREATE TABLE dbo.probe_vs_fk_c (v vector(3) NOT NULL, CONSTRAINT fk_probe_vs FOREIGN KEY (v) REFERENCES dbo.probe_vs_fk_p(v));", + "expect": { + "error": 1776 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A vector column cannot be a candidate key, so nothing can reference it. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_check (id int, v vector(3) NULL, CONSTRAINT ck_probe_vs CHECK (v IS NOT NULL));", + "expect": { + "error": 1760 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The restriction list says a CHECK constraint on a vector column is refused. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_default (id int, v vector(3) NOT NULL CONSTRAINT df_probe_vs DEFAULT ('[1,2,3]'));", + "expect": { + "error": 1752 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The restriction list says a DEFAULT constraint on a vector column is refused. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_persisted (id int, v vector(3) NOT NULL, d AS VECTOR_DISTANCE('cosine', v, CAST(N'[1,2,3]' AS vector(3))) PERSISTED);", + "expect": { + "error": 4936 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A persisted computed column over a vector function is refused as non-deterministic. A non-persisted one is allowed, which is the distinction the skill draws. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE PARTITION FUNCTION pf_probe_vs (vector(3)) AS RANGE LEFT FOR VALUES ('[1,2,3]');", + "expect": { + "error": 7704 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A vector column cannot be a partition key. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TYPE dbo.probe_vs_alias FROM vector(3);", + "expect": { + "error": 42212 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. No alias type over vector, so a shop that wraps every domain type in CREATE TYPE cannot wrap this one. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "DECLARE @sv sql_variant;\nDECLARE @v vector(3) = N'[1,2,3]';\nSET @sv = @v;\nSELECT @sv AS x;", + "expect": { + "error": 206 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Assignment to sql_variant is an operand type clash, so a generic property bag cannot hold an embedding. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_altercol (id int, v vector(3) NOT NULL);\nALTER TABLE dbo.probe_vs_altercol ALTER COLUMN v vector(5) NOT NULL;", + "expect": { + "error": 42204 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The dimension of an existing vector column cannot be changed by ALTER COLUMN, and this table is empty when the ALTER runs, which is the case readers assume is allowed. It is one of the five states the value probe asks the model to assert. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_orderby (id int, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_orderby VALUES (1, N'[1,2,3]');\nSELECT id FROM dbo.probe_vs_orderby ORDER BY v;", + "expect": { + "error": 42213 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A vector column cannot be sorted, which is separate from ORDER BY on a distance expression over it. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_distinct (id int, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_distinct VALUES (1, N'[1,2,3]');\nSELECT DISTINCT v FROM dbo.probe_vs_distinct;", + "expect": { + "error": 421 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. SELECT DISTINCT over the column is refused, the second half of why deduplication cannot key off the embedding. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_union (id int, v vector(3) NOT NULL);\nSELECT v FROM dbo.probe_vs_union UNION SELECT v FROM dbo.probe_vs_union;", + "expect": { + "error": 5335 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. UNION, INTERSECT and EXCEPT are refused over a vector column while UNION ALL is allowed, which is the distinction the skill draws. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_stats (id int, v vector(3) NOT NULL);\nCREATE STATISTICS st_probe_vs ON dbo.probe_vs_stats (v);", + "expect": { + "error": 1978 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A vector column cannot be a statistics target, so the statistics remedy an agent proposes for slow similarity search cannot even be created. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "DECLARE @v vector(3) = N'[1,2,3]';\nSELECT CAST(@v AS varbinary(100)) AS b;", + "expect": { + "error": 529 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Explicit conversion to varbinary is refused, so an embedding cannot be smuggled out as bytes. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_wronglen (id int, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_wronglen VALUES (1, N'[1,2,3,4]');", + "expect": { + "error": 42204 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Inserting a JSON array of the wrong length is refused, which is what catches an embedding model swap at write time rather than at read time. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_jsonstr (id int, v vector(3) NOT NULL);\nINSERT INTO dbo.probe_vs_jsonstr VALUES (1, N'[1,\"a\",3]');", + "expect": { + "error": 13670 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A JSON array containing a string is not a valid vector, which is the failure a hand-built payload hits. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "DECLARE @a vector(3) = N'[1,2,3]', @b vector(3) = N'[1,2,4]';\nSELECT VECTOR_DISTANCE('manhattan', @a, @b) AS d;", + "expect": { + "error": 42201 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Three metrics exist and nothing else does, so a metric name carried over from another vector store is refused rather than approximated. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "DECLARE @a vector(3, float16) = N'[1,2,3]';\nDECLARE @b vector(3) = N'[1,2,4]';\nSELECT VECTOR_DISTANCE('cosine', @a, @b) AS d;", + "expect": { + "error": 42243 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. Both arguments must share a base type, so a half precision column cannot be compared against a float32 query vector. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_approx_bare (id int NOT NULL PRIMARY KEY CLUSTERED, v vector(3) NOT NULL);\nSELECT TOP (2) WITH APPROXIMATE id FROM dbo.probe_vs_approx_bare ORDER BY id;", + "expect": { + "error": 42248 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. WITH APPROXIMATE is not a general modifier and is refused without a VECTOR_SEARCH, which is the shape an agent reaches for when told to make a TOP query approximate. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_second (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_second (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_second_ix ON dbo.probe_vs_second (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nCREATE VECTOR INDEX probe_vs_second_ix2 ON dbo.probe_vs_second (v) WITH (METRIC = 'euclidean', TYPE = 'diskann');", + "expect": { + "error": 42230 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. One vector index per column, even with a different metric, which is why a second metric means a second table rather than a second index. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_rebuild (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_rebuild (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_rebuild_ix ON dbo.probe_vs_rebuild (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nALTER INDEX probe_vs_rebuild_ix ON dbo.probe_vs_rebuild REBUILD;", + "expect": { + "error": 42250 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. There is no rebuild for a vector index, so the maintenance job every DBA already has will fail on this table. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_dropcol (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_dropcol (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_dropcol_ix ON dbo.probe_vs_dropcol (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nALTER TABLE dbo.probe_vs_dropcol DROP COLUMN v;", + "expect": { + "error": 5074 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The index holds the column, so a migration that drops it has to drop the index first. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_sortkey (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_sortkey (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_sortkey_ix ON dbo.probe_vs_sortkey (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(5) = N'[0.3,0.3,0.3,0.3,0.3]';\nSELECT TOP (3) WITH APPROXIMATE t.id, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_vs_sortkey AS t, COLUMN = v, SIMILAR_TO = @q, METRIC = 'cosine') AS r\nORDER BY r.distance, t.id;", + "expect": { + "error": 42271 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The ORDER BY of an approximate search is fixed to the distance column ascending and nothing else. Microsoft Learn's own end to end example on the CREATE VECTOR INDEX page adds a second sort key, so this is the number a reader hits by copying the documentation. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_heap (id int NOT NULL, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_heap (id, v)\nSELECT value, CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_heap_ix ON dbo.probe_vs_heap (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "error": 42254 + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. A heap cannot carry a vector index, so the clustered index is a prerequisite and not a recommendation. It goes red if the statement succeeds, or fails with any other number, which is what would happen if the engine relaxed this restriction or renumbered it." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_fallback (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_fallback (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_fallback_ix ON dbo.probe_vs_fallback (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(5) = N'[0.3,0.3,0.3,0.3,0.3]';\nSELECT COUNT(*) AS rows_returned FROM (\n SELECT TOP (3) WITH APPROXIMATE t.id, r.distance\n FROM VECTOR_SEARCH(TABLE = dbo.probe_vs_fallback AS t, COLUMN = v, SIMILAR_TO = @q, METRIC = 'euclidean') AS r\n ORDER BY r.distance\n) AS s;", + "expect": { + "rows": [ + [ + 3 + ] + ] + }, + "why": "THE SILENT FALLBACK, and the correction this probe was written to hold. The index above is built on 'cosine'. Asking VECTOR_SEARCH for 'euclidean' returns a full ordered result set with no error and no warning: measured 2026-09-06 on the container at 18.0.226_4_147. This skill said Msg 42227 here until that measurement, which was backwards in the one direction that matters, because a reader who believes the engine will stop them never checks. It goes red if the statement raises anything, including Msg 42227, or returns any number of rows other than 3." + }, + { + "sql": "CREATE TABLE dbo.probe_vs_forceann_metric (id int IDENTITY(1,1) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_forceann_metric (v)\nSELECT CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_forceann_metric_ix ON dbo.probe_vs_forceann_metric (v) WITH (METRIC = 'cosine', TYPE = 'diskann');\nGO\nDECLARE @q vector(5) = N'[0.3,0.3,0.3,0.3,0.3]';\nSELECT TOP (3) WITH APPROXIMATE t.id, r.distance\nFROM VECTOR_SEARCH(TABLE = dbo.probe_vs_forceann_metric AS t, COLUMN = v, SIMILAR_TO = @q, METRIC = 'euclidean') AS r WITH (FORCE_ANN_ONLY)\nORDER BY r.distance;", + "expect": { + "error": 42227 + }, + "why": "The other half of the pair above, and the reason FORCE_ANN_ONLY belongs in a test. The same query that falls back silently raises Msg 42227 the moment the fallback is forbidden, so this hint is the only thing that turns a metric mismatch into an error a pipeline can see. Measured 2026-09-06. It goes red if the statement succeeds, which would mean FORCE_ANN_ONLY had stopped being a guarantee." + } + , + { + "sql": "CREATE TABLE dbo.probe_vs_nvpk (id nvarchar(40) NOT NULL PRIMARY KEY CLUSTERED, v vector(5) NOT NULL);\nINSERT INTO dbo.probe_vs_nvpk (id, v)\nSELECT CAST(value AS nvarchar(40)), CAST(JSON_ARRAY(CAST(value * 0.01 AS float), CAST(value * 0.02 AS float), CAST(value * 0.03 AS float), CAST(value * 0.04 AS float), CAST(value * 0.05 AS float)) AS vector(5))\nFROM GENERATE_SERIES(1, 150);\nGO\nCREATE VECTOR INDEX probe_vs_nvpk_ix ON dbo.probe_vs_nvpk (v) WITH (METRIC = 'cosine', TYPE = 'diskann');", + "expect": { + "succeeds": true + }, + "why": "Measured 2026-09-06 on the container at 18.0.226_4_147, EngineEdition 5. The one claim in this skill that is an ABSENCE: Msg 42217's narrower wording says the clustered index has to be a primary key on particular types, and the skill calls that stale. An nvarchar clustered primary key builds the index, so the narrower rule is not the rule. It goes red if the build ever raises, which is what would happen if 42217's wording became true." + } + ] + }, + "triggering": { + "explicit": [ + "Use the vector-search-azure-sql skill", + "Add a vector column to my Azure SQL Database table and search it", + "Create a vector index on my embeddings column in Azure SQL Database", + "How do I run a top k nearest neighbour query on Azure SQL Database", + "What are the restrictions on the vector data type in Azure SQL Database" + ], + "implicit": [ + "my similarity search returns the right rows but takes seconds and gets slower every week", + "why can I not put a primary key or a unique constraint on my embedding column", + "Vector search with newer index version does not support explicit TOP_N parameter", + "The vector data types cannot be compared or sorted, except when using the IS NULL operator", + "The size (3072) given to the column exceeds the maximum allowed", + "I cannot truncate my table any more since I added the vector index", + "is my query actually using the vector index or is it scanning", + "how do I deduplicate rows that have the same embedding", + "do I have to turn on PREVIEW_FEATURES before CREATE VECTOR INDEX will work" + ], + "routing": [ + "store OpenAI embeddings in Azure SQL Database and query them", + "replace my separate vector database with Azure SQL Database", + "make my nearest neighbour query on Azure SQL Database faster" + ], + "negative": [ + "Build a retrieval augmented generation pipeline over my document library on Azure SQL Database", + "Chunk my documents and keep the embeddings up to date as the source changes", + "Generate embeddings with an external model from inside the database", + "Create an external model and call AI_GENERATE_EMBEDDINGS", + "Run a local RAG prototype against the Azure SQL Database container with a local embedding model", + "Build a RAG scenario on the container", + "Design the schema for my orders and customers tables", + "Add retry and connection pooling to my data access layer", + "Set up full text search and rank the results" + ] + } +} diff --git a/scripts/generate.mjs b/scripts/generate.mjs index a533ff3..c93f748 100644 --- a/scripts/generate.mjs +++ b/scripts/generate.mjs @@ -299,7 +299,7 @@ emit('.claude-plugin/plugin.json', j({ displayName: STORE_NAME, ...base })); // Nothing else changes between the two, so the clients can be measured against // the same manifests. // --------------------------------------------------------------------------- -const SUBSET_SHARING = 'symlink'; +const SUBSET_SHARING = 'copy'; // Data API builder is excluded from the Visual Studio Code plugin because it // clashes with Data API builder tooling GitHub Copilot already carries there. From 6a8a3a7346e48b410843bcc8042c47ec671e298c Mon Sep 17 00:00:00 2001 From: Carlos Robles Date: Mon, 21 Sep 2026 22:27:42 -0600 Subject: [PATCH 3/3] Record what each client did with the symlinks --- scripts/generate.mjs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/generate.mjs b/scripts/generate.mjs index c93f748..73ab938 100644 --- a/scripts/generate.mjs +++ b/scripts/generate.mjs @@ -298,6 +298,31 @@ emit('.claude-plugin/plugin.json', j({ displayName: STORE_NAME, ...base })); // 'copy' a byte-for-byte copy, regenerated here and diffed by --check // Nothing else changes between the two, so the clients can be measured against // the same manifests. +// +// MEASURED 2026-09-21, installing this branch from git into each client: +// +// symlink Claude Code 20 / 8 / 4 / 6, correct. It dereferences the link +// into the cache, exactly as its documentation says. +// GitHub Copilot 20 / 8 / 4 / 6, correct, dereferenced the same way. +// Codex ZERO. "Added plugin" and an empty skills/ in the +// plugin cache. Its git clone keeps the links and its +// copy into the cache drops every one, with no error +// anywhere. An install that reports success and loads +// nothing is the exact failure this project keeps +// finding late, so symlinks are out. +// +// Also out on Windows. `git clone -c core.symlinks=false`, which is the +// default without the create-symlink privilege, writes each link as a 39-byte +// text file holding the path. No SKILL.md under any of them. +// +// copy Claude Code, GitHub Copilot and Codex all load 20 / 8 / 4 / 6 with +// the Data API builder skills absent from the Visual Studio Code +// plugin. Cursor is UNMEASURED: no Cursor agent CLI on the machine. +// +// So the content exists four more times on disk, and the repository still has +// exactly one place a human edits: skills/. The --check arm above diffs every +// copy against its source, which was proved by editing one copy by hand and +// watching the gate fail. // --------------------------------------------------------------------------- const SUBSET_SHARING = 'copy';