From c92b259a7699419263e1935280aeefcd1e678edd Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 12:19:08 -0700 Subject: [PATCH 01/19] update nuget packages and initial code implementation --- .../GenerateBatchVmSkus.csproj | 2 +- src/Tes.Runner.Test/Tes.Runner.Test.csproj | 2 +- src/Tes/Repository/TesDbContext.cs | 25 +++++++++++++++++-- src/Tes/Tes.csproj | 9 ++++--- src/TesApi.Web/TesApi.Web.csproj | 2 +- .../deploy-tes-on-azure.csproj | 2 +- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj b/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj index f672a4388..d66f38403 100644 --- a/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj +++ b/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Tes.Runner.Test/Tes.Runner.Test.csproj b/src/Tes.Runner.Test/Tes.Runner.Test.csproj index c31072576..dabfb5d85 100644 --- a/src/Tes.Runner.Test/Tes.Runner.Test.csproj +++ b/src/Tes.Runner.Test/Tes.Runner.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index aa766ba61..6df0bbc77 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Text; +using Azure.Identity; using Microsoft.EntityFrameworkCore; using Tes.Models; @@ -9,6 +11,7 @@ namespace Tes.Repository { public class TesDbContext : DbContext { + private const string aadResourceId = "https://ossrdbms-aad.database.windows.net/.default"; public const string TesTasksPostgresTableName = "testasks"; public TesDbContext() @@ -26,11 +29,29 @@ public TesDbContext(string connectionString) public string ConnectionString { get; set; } public DbSet TesTasks { get; set; } - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { if (!optionsBuilder.IsConfigured) { - // use PostgreSQL + if (ConnectionString.Contains("PASSWORD=CLIENT_ID;", StringComparison.OrdinalIgnoreCase)) + { + // Use AAD managed identity (https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity) + // Instructions: + // 1. You must run this on your PostgreSQL server, replacing 'myuser' with the correct user: + /* + SET aad_validate_oids_in_tenant = off; + CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; + */ + // 2. You must set DatabaseUserPassword to "CLIENT_ID" in the TES AKS configuration + + // Note: this supports token caching internally + var credential = new DefaultAzureCredential(); + var accessToken = await credential.GetTokenAsync( + new Azure.Core.TokenRequestContext(scopes: new string[] { aadResourceId })); + + ConnectionString.Replace("PASSWORD=CLIENT_ID;", $"PASSWORD={accessToken.Token};"); + } + optionsBuilder .UseNpgsql(ConnectionString, options => options.MaxBatchSize(1000)) .UseLowerCaseNamingConvention(); diff --git a/src/Tes/Tes.csproj b/src/Tes/Tes.csproj index ab248c35b..620aeb0ee 100644 --- a/src/Tes/Tes.csproj +++ b/src/Tes/Tes.csproj @@ -5,18 +5,19 @@ + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index 8167d1e4e..7d0b7268c 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj b/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj index 6bdef9591..932a376e5 100644 --- a/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj +++ b/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj @@ -26,7 +26,7 @@ - + From 3559b8b5cd4e3b8e4923055de9a840db9df2265b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 12:26:33 -0700 Subject: [PATCH 02/19] refactor --- src/Tes/Repository/TesDbContext.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 6df0bbc77..13c4088e1 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -11,7 +11,8 @@ namespace Tes.Repository { public class TesDbContext : DbContext { - private const string aadResourceId = "https://ossrdbms-aad.database.windows.net/.default"; + private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; + private const string defaultManagedIdentityPassword = "CLIENT_ID"; public const string TesTasksPostgresTableName = "testasks"; public TesDbContext() @@ -33,23 +34,26 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild { if (!optionsBuilder.IsConfigured) { - if (ConnectionString.Contains("PASSWORD=CLIENT_ID;", StringComparison.OrdinalIgnoreCase)) + string connectionStringTargetReplacement = $"PASSWORD={defaultManagedIdentityPassword};"; + + if (ConnectionString.Contains(connectionStringTargetReplacement, StringComparison.OrdinalIgnoreCase)) { - // Use AAD managed identity (https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity) + // Use AAD managed identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity // Instructions: - // 1. You must run this on your PostgreSQL server, replacing 'myuser' with the correct user: + // 1. Replace 'myuser' and run on your server /* SET aad_validate_oids_in_tenant = off; CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; */ - // 2. You must set DatabaseUserPassword to "CLIENT_ID" in the TES AKS configuration + // 2. Set "DatabaseUserPassword" to "CLIENT_ID" in the TES AKS configuration // Note: this supports token caching internally var credential = new DefaultAzureCredential(); var accessToken = await credential.GetTokenAsync( - new Azure.Core.TokenRequestContext(scopes: new string[] { aadResourceId })); + new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - ConnectionString.Replace("PASSWORD=CLIENT_ID;", $"PASSWORD={accessToken.Token};"); + ConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); } optionsBuilder From ff1736c4bb9251ad3929e38cdb542845fbe6809a Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:18:05 -0700 Subject: [PATCH 03/19] update logic --- src/Tes/Repository/TesDbContext.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 13c4088e1..dff7caadd 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -14,6 +14,7 @@ public class TesDbContext : DbContext private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; private const string defaultManagedIdentityPassword = "CLIENT_ID"; public const string TesTasksPostgresTableName = "testasks"; + private static DateTimeOffset accessTokenLastExpiration = DateTimeOffset.MinValue; public TesDbContext() { @@ -34,12 +35,15 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild { if (!optionsBuilder.IsConfigured) { + string tempConnectionString = ConnectionString; + string connectionStringTargetReplacement = $"PASSWORD={defaultManagedIdentityPassword};"; - if (ConnectionString.Contains(connectionStringTargetReplacement, StringComparison.OrdinalIgnoreCase)) + if (tempConnectionString.Contains(connectionStringTargetReplacement, StringComparison.OrdinalIgnoreCase)) { // Use AAD managed identity // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/concepts-azure-ad-authentication // Instructions: // 1. Replace 'myuser' and run on your server /* @@ -47,17 +51,21 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; */ // 2. Set "DatabaseUserPassword" to "CLIENT_ID" in the TES AKS configuration + // 3. Ensure the managed identity that TES runs under has the role // Note: this supports token caching internally var credential = new DefaultAzureCredential(); var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - - ConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); + + if (accessToken.ExpiresOn != accessTokenLastExpiration) + { + tempConnectionString = tempConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); + } } optionsBuilder - .UseNpgsql(ConnectionString, options => options.MaxBatchSize(1000)) + .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(1000)) .UseLowerCaseNamingConvention(); } } From 820c2e05f58b7c330a890ec345897a9b93002896 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:18:40 -0700 Subject: [PATCH 04/19] refactor --- src/Tes/Repository/TesDbContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index dff7caadd..d15d6c067 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -36,7 +36,6 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild if (!optionsBuilder.IsConfigured) { string tempConnectionString = ConnectionString; - string connectionStringTargetReplacement = $"PASSWORD={defaultManagedIdentityPassword};"; if (tempConnectionString.Contains(connectionStringTargetReplacement, StringComparison.OrdinalIgnoreCase)) From ecc585615d9f06036bf27e89162973a5d95e82c5 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:21:16 -0700 Subject: [PATCH 05/19] fix connection string logic --- src/Tes/Repository/TesDbContext.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index d15d6c067..0a0411bb6 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -14,7 +14,6 @@ public class TesDbContext : DbContext private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; private const string defaultManagedIdentityPassword = "CLIENT_ID"; public const string TesTasksPostgresTableName = "testasks"; - private static DateTimeOffset accessTokenLastExpiration = DateTimeOffset.MinValue; public TesDbContext() { @@ -57,10 +56,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - if (accessToken.ExpiresOn != accessTokenLastExpiration) - { - tempConnectionString = tempConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); - } + tempConnectionString = tempConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); } optionsBuilder From 7795580152148b092e2f1daa94cd8184b5291d99 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:23:30 -0700 Subject: [PATCH 06/19] rewrite for clarity --- src/Tes/Repository/TesDbContext.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 0a0411bb6..bb2f4f6c5 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -35,9 +35,10 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild if (!optionsBuilder.IsConfigured) { string tempConnectionString = ConnectionString; - string connectionStringTargetReplacement = $"PASSWORD={defaultManagedIdentityPassword};"; + string managedIdentityPasswordSetting = $"PASSWORD={defaultManagedIdentityPassword};"; + bool isManagedIdentityEnabled = tempConnectionString.Contains(managedIdentityPasswordSetting, StringComparison.OrdinalIgnoreCase); - if (tempConnectionString.Contains(connectionStringTargetReplacement, StringComparison.OrdinalIgnoreCase)) + if (isManagedIdentityEnabled) { // Use AAD managed identity // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity @@ -56,7 +57,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - tempConnectionString = tempConnectionString.Replace(connectionStringTargetReplacement, $"PASSWORD={accessToken.Token};"); + tempConnectionString = tempConnectionString.Replace(managedIdentityPasswordSetting, $"PASSWORD={accessToken.Token};"); } optionsBuilder From 01dd40a236cd05181ce6a573c67a5abd8b8da4b8 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:23:55 -0700 Subject: [PATCH 07/19] refactor --- src/Tes/Repository/TesDbContext.cs | 56 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index bb2f4f6c5..135aeb3c8 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -32,38 +32,40 @@ public TesDbContext(string connectionString) protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (!optionsBuilder.IsConfigured) + if (optionsBuilder.IsConfigured) { - string tempConnectionString = ConnectionString; - string managedIdentityPasswordSetting = $"PASSWORD={defaultManagedIdentityPassword};"; - bool isManagedIdentityEnabled = tempConnectionString.Contains(managedIdentityPasswordSetting, StringComparison.OrdinalIgnoreCase); + return; + } + + string tempConnectionString = ConnectionString; + string managedIdentityPasswordSetting = $"PASSWORD={defaultManagedIdentityPassword};"; + bool isManagedIdentityEnabled = tempConnectionString.Contains(managedIdentityPasswordSetting, StringComparison.OrdinalIgnoreCase); - if (isManagedIdentityEnabled) - { - // Use AAD managed identity - // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity - // https://learn.microsoft.com/en-us/azure/postgresql/single-server/concepts-azure-ad-authentication - // Instructions: - // 1. Replace 'myuser' and run on your server - /* - SET aad_validate_oids_in_tenant = off; - CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; - */ - // 2. Set "DatabaseUserPassword" to "CLIENT_ID" in the TES AKS configuration - // 3. Ensure the managed identity that TES runs under has the role + if (isManagedIdentityEnabled) + { + // Use AAD managed identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/concepts-azure-ad-authentication + // Instructions: + // 1. Replace 'myuser' and run on your server + /* + SET aad_validate_oids_in_tenant = off; + CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; + */ + // 2. Set "DatabaseUserPassword" to "CLIENT_ID" in the TES AKS configuration + // 3. Ensure the managed identity that TES runs under has the role - // Note: this supports token caching internally - var credential = new DefaultAzureCredential(); - var accessToken = await credential.GetTokenAsync( - new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); + // Note: this supports token caching internally + var credential = new DefaultAzureCredential(); + var accessToken = await credential.GetTokenAsync( + new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - tempConnectionString = tempConnectionString.Replace(managedIdentityPasswordSetting, $"PASSWORD={accessToken.Token};"); - } - - optionsBuilder - .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(1000)) - .UseLowerCaseNamingConvention(); + tempConnectionString = tempConnectionString.Replace(managedIdentityPasswordSetting, $"PASSWORD={accessToken.Token};"); } + + optionsBuilder + .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(1000)) + .UseLowerCaseNamingConvention(); } } } From cf722640ab2611252a287dab3fb2d30ac30bab21 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:47:24 -0700 Subject: [PATCH 08/19] add new setting --- src/Tes/Models/PostgreSqlOptions.cs | 1 + src/Tes/Repository/TesDbContext.cs | 23 +++++++------------ .../Repository/TesTaskPostgreSqlRepository.cs | 2 +- .../PostgresConnectionStringUtility.cs | 13 +++++++++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Tes/Models/PostgreSqlOptions.cs b/src/Tes/Models/PostgreSqlOptions.cs index 65ef70e46..0fe19f255 100644 --- a/src/Tes/Models/PostgreSqlOptions.cs +++ b/src/Tes/Models/PostgreSqlOptions.cs @@ -27,5 +27,6 @@ public static string GetConfigurationSectionName(string serviceName = "Tes") public string DatabaseName { get; set; } = "tes_db"; public string DatabaseUserLogin { get; set; } public string DatabaseUserPassword { get; set; } + public bool UseManagedIdentity { get; set; } } } diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 135aeb3c8..d09db5a5b 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Text; using Azure.Identity; using Microsoft.EntityFrameworkCore; using Tes.Models; @@ -12,8 +11,8 @@ namespace Tes.Repository public class TesDbContext : DbContext { private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; - private const string defaultManagedIdentityPassword = "CLIENT_ID"; public const string TesTasksPostgresTableName = "testasks"; + public bool UseManagedIdentity { get; set; } public TesDbContext() { @@ -21,27 +20,21 @@ public TesDbContext() // "dotnet ef migrations add InitialCreate" } - public TesDbContext(string connectionString) + public TesDbContext(string connectionString, bool useManagedIdentity = false) { ArgumentException.ThrowIfNullOrEmpty(connectionString, nameof(connectionString)); ConnectionString = connectionString; + UseManagedIdentity = useManagedIdentity; } public string ConnectionString { get; set; } public DbSet TesTasks { get; set; } protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - if (optionsBuilder.IsConfigured) - { - return; - } - + { string tempConnectionString = ConnectionString; - string managedIdentityPasswordSetting = $"PASSWORD={defaultManagedIdentityPassword};"; - bool isManagedIdentityEnabled = tempConnectionString.Contains(managedIdentityPasswordSetting, StringComparison.OrdinalIgnoreCase); - if (isManagedIdentityEnabled) + if (UseManagedIdentity) { // Use AAD managed identity // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity @@ -52,15 +45,15 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild SET aad_validate_oids_in_tenant = off; CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; */ - // 2. Set "DatabaseUserPassword" to "CLIENT_ID" in the TES AKS configuration + // 2. Set "PostgreSql.UseManagedIdentity" to "true" in the TES AKS configuration // 3. Ensure the managed identity that TES runs under has the role // Note: this supports token caching internally var credential = new DefaultAzureCredential(); var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - - tempConnectionString = tempConnectionString.Replace(managedIdentityPasswordSetting, $"PASSWORD={accessToken.Token};"); + + tempConnectionString = tempConnectionString.TrimEnd(';') + $"PASSWORD={accessToken.Token};"; } optionsBuilder diff --git a/src/Tes/Repository/TesTaskPostgreSqlRepository.cs b/src/Tes/Repository/TesTaskPostgreSqlRepository.cs index 709f6d635..a0cce8e7d 100644 --- a/src/Tes/Repository/TesTaskPostgreSqlRepository.cs +++ b/src/Tes/Repository/TesTaskPostgreSqlRepository.cs @@ -33,7 +33,7 @@ public TesTaskPostgreSqlRepository(IOptions options, ILogger< : base(logger, cache) { var connectionString = new ConnectionStringUtility().GetPostgresConnectionString(options); - CreateDbContext = () => { return new TesDbContext(connectionString); }; + CreateDbContext = () => { return new TesDbContext(connectionString, options.Value.UseManagedIdentity); }; using var dbContext = CreateDbContext(); dbContext.Database.MigrateAsync().Wait(); WarmCacheAsync(CancellationToken.None).Wait(); diff --git a/src/Tes/Utilities/PostgresConnectionStringUtility.cs b/src/Tes/Utilities/PostgresConnectionStringUtility.cs index e5263e47d..180d66541 100644 --- a/src/Tes/Utilities/PostgresConnectionStringUtility.cs +++ b/src/Tes/Utilities/PostgresConnectionStringUtility.cs @@ -18,7 +18,11 @@ public string GetPostgresConnectionString(IOptions options) ArgumentException.ThrowIfNullOrEmpty(options.Value.ServerSslMode, nameof(options.Value.ServerSslMode)); ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseName, nameof(options.Value.DatabaseName)); ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserLogin, nameof(options.Value.DatabaseUserLogin)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserPassword, nameof(options.Value.DatabaseUserPassword)); + + if (!options.Value.UseManagedIdentity) + { + ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserPassword, nameof(options.Value.DatabaseUserPassword)); + } if (options.Value.ServerName.Contains(options.Value.ServerNameSuffix, StringComparison.OrdinalIgnoreCase)) { @@ -30,7 +34,12 @@ public string GetPostgresConnectionString(IOptions options) connectionStringBuilder.Append($"Database={options.Value.DatabaseName};"); connectionStringBuilder.Append($"Port={options.Value.ServerPort};"); connectionStringBuilder.Append($"User Id={options.Value.DatabaseUserLogin};"); - connectionStringBuilder.Append($"Password={options.Value.DatabaseUserPassword};"); + + if (!options.Value.UseManagedIdentity) + { + connectionStringBuilder.Append($"Password={options.Value.DatabaseUserPassword};"); + } + connectionStringBuilder.Append($"SSL Mode={options.Value.ServerSslMode};"); return connectionStringBuilder.ToString(); } From e4ccb523ee16e20946e0293bfc6139ae7c103185 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:48:40 -0700 Subject: [PATCH 09/19] fix semicolon --- src/Tes/Repository/TesDbContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index d09db5a5b..7de638793 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -53,7 +53,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - tempConnectionString = tempConnectionString.TrimEnd(';') + $"PASSWORD={accessToken.Token};"; + tempConnectionString = tempConnectionString.TrimEnd(';') + $";PASSWORD={accessToken.Token};"; } optionsBuilder From 9335a3e9de08e20b00ab3c2e87da6839d8633439 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:49:52 -0700 Subject: [PATCH 10/19] add note --- src/Tes/Repository/TesDbContext.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 7de638793..1b2709f40 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -53,6 +53,8 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); + // ConnectionStringUtility omits password when UseManagedIdentity is set. + // Omitting an assertion here to avoid the performance hit of string comparison on every creation tempConnectionString = tempConnectionString.TrimEnd(';') + $";PASSWORD={accessToken.Token};"; } From 495486907dd18484dfbda19506e3aa4ede24fbcf Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:55:26 -0700 Subject: [PATCH 11/19] add additional argument exception check --- src/Tes/Utilities/PostgresConnectionStringUtility.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Tes/Utilities/PostgresConnectionStringUtility.cs b/src/Tes/Utilities/PostgresConnectionStringUtility.cs index 180d66541..96e1ae6f3 100644 --- a/src/Tes/Utilities/PostgresConnectionStringUtility.cs +++ b/src/Tes/Utilities/PostgresConnectionStringUtility.cs @@ -21,9 +21,16 @@ public string GetPostgresConnectionString(IOptions options) if (!options.Value.UseManagedIdentity) { + // Ensure password is set if NOT using Managed Identity ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserPassword, nameof(options.Value.DatabaseUserPassword)); } + if (options.Value.UseManagedIdentity && !string.IsNullOrWhiteSpace(options.Value.DatabaseUserPassword)) + { + // throw if password IS set when using Managed Identity + throw new ArgumentException("DatabaseUserPassword shall not be set if UseManagedIdentity is true"); + } + if (options.Value.ServerName.Contains(options.Value.ServerNameSuffix, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException($"'{nameof(options.Value.ServerName)}' should only contain the name of the server like 'myserver' and NOT the full host name like 'myserver{options.Value.ServerNameSuffix}'", nameof(options.Value.ServerName)); From df02e60bc64a767720775f9720ff9f4d83baae98 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:56:01 -0700 Subject: [PATCH 12/19] update casing --- src/Tes/Repository/TesDbContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 1b2709f40..3857b90a1 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -55,7 +55,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild // ConnectionStringUtility omits password when UseManagedIdentity is set. // Omitting an assertion here to avoid the performance hit of string comparison on every creation - tempConnectionString = tempConnectionString.TrimEnd(';') + $";PASSWORD={accessToken.Token};"; + tempConnectionString = tempConnectionString.TrimEnd(';') + $";Password={accessToken.Token};"; } optionsBuilder From 6c827f13d51a13e8e168920805faf06ff8147a28 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 13:58:02 -0700 Subject: [PATCH 13/19] add note about connection string check --- src/Tes/Repository/TesDbContext.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 3857b90a1..57e7ff691 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -53,8 +53,10 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild var accessToken = await credential.GetTokenAsync( new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - // ConnectionStringUtility omits password when UseManagedIdentity is set. - // Omitting an assertion here to avoid the performance hit of string comparison on every creation + // ConnectionStringUtility omits password when UseManagedIdentity is set, therefore + // omitting this to avoid performance hit of string comparison on every creation + // if (tempConnectionString.Contains("Password=", StringComparison.OrdinalIgnoreCase)) throw new Exception("Password shall not be provided when using managed identity"); + tempConnectionString = tempConnectionString.TrimEnd(';') + $";Password={accessToken.Token};"; } From 1022f253b756ae742d5432317e621f0441abe3d8 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 15:56:18 -0700 Subject: [PATCH 14/19] fix formatting --- src/Tes/Repository/TesDbContext.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 57e7ff691..71ab5f7e9 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -12,7 +12,7 @@ public class TesDbContext : DbContext { private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; public const string TesTasksPostgresTableName = "testasks"; - public bool UseManagedIdentity { get; set; } + public bool UseManagedIdentity { get; set; } public TesDbContext() { @@ -31,7 +31,7 @@ public TesDbContext(string connectionString, bool useManagedIdentity = false) public DbSet TesTasks { get; set; } protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { + { string tempConnectionString = ConnectionString; if (UseManagedIdentity) From e7722b0ba0d1688365f041fc7eb17c3a4552f249 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 15:57:48 -0700 Subject: [PATCH 15/19] hoist maxbatchsize --- src/Tes/Repository/TesDbContext.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 71ab5f7e9..33fe98de3 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -10,6 +10,7 @@ namespace Tes.Repository { public class TesDbContext : DbContext { + private const int maxBatchSize = 1000; private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; public const string TesTasksPostgresTableName = "testasks"; public bool UseManagedIdentity { get; set; } @@ -61,7 +62,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild } optionsBuilder - .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(1000)) + .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(maxBatchSize)) .UseLowerCaseNamingConvention(); } } From 7eba550e246138f6a8552f743b834248f0780dcb Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 16:46:13 -0700 Subject: [PATCH 16/19] minor formatting --- src/Tes/Repository/TesDbContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 33fe98de3..92aab4caf 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -58,7 +58,7 @@ protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuild // omitting this to avoid performance hit of string comparison on every creation // if (tempConnectionString.Contains("Password=", StringComparison.OrdinalIgnoreCase)) throw new Exception("Password shall not be provided when using managed identity"); - tempConnectionString = tempConnectionString.TrimEnd(';') + $";Password={accessToken.Token};"; + tempConnectionString = $"{tempConnectionString.TrimEnd(';')};Password={accessToken.Token};"; } optionsBuilder From 9bf1c3eb70dd7818684e269fad3eeb99df701305 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 22 Aug 2023 09:30:28 -0700 Subject: [PATCH 17/19] add Microsoft.EntityFrameworkCore.Relational and update packages --- src/Tes.Runner.Test/Tes.Runner.Test.csproj | 2 +- src/Tes/Tes.csproj | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Tes.Runner.Test/Tes.Runner.Test.csproj b/src/Tes.Runner.Test/Tes.Runner.Test.csproj index dabfb5d85..29f58c949 100644 --- a/src/Tes.Runner.Test/Tes.Runner.Test.csproj +++ b/src/Tes.Runner.Test/Tes.Runner.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Tes/Tes.csproj b/src/Tes/Tes.csproj index 620aeb0ee..2448667de 100644 --- a/src/Tes/Tes.csproj +++ b/src/Tes/Tes.csproj @@ -7,12 +7,13 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + + From 6441716781b46a76863e77f3a200711a720fb017 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 10 Oct 2023 16:25:24 -0700 Subject: [PATCH 18/19] update implementation and refactor dbcontext --- src/Tes/Models/TesTaskPostgres.cs | 2 +- .../Repository/PostgreSqlCachingRepository.cs | 10 ++- src/Tes/Repository/TesDbContext.cs | 47 +++--------- .../Repository/TesTaskPostgreSqlRepository.cs | 30 +++----- .../PostgresConnectionStringUtility.cs | 74 +++++++++++++------ ...askPostgreSqlRepositoryIntegrationTests.cs | 7 +- src/TesApi.Web/Startup.cs | 6 +- 7 files changed, 87 insertions(+), 89 deletions(-) diff --git a/src/Tes/Models/TesTaskPostgres.cs b/src/Tes/Models/TesTaskPostgres.cs index bc6a528ac..6a7ed97aa 100644 --- a/src/Tes/Models/TesTaskPostgres.cs +++ b/src/Tes/Models/TesTaskPostgres.cs @@ -9,7 +9,7 @@ namespace Tes.Models /// /// Database schema for encapsulating a TesTask as Json for Postgresql. /// - [Table(Repository.TesDbContext.TesTasksPostgresTableName)] + [Table("testasks")] public class TesTaskDatabaseItem { [Column("id")] diff --git a/src/Tes/Repository/PostgreSqlCachingRepository.cs b/src/Tes/Repository/PostgreSqlCachingRepository.cs index 8a8c36811..e069fdde1 100644 --- a/src/Tes/Repository/PostgreSqlCachingRepository.cs +++ b/src/Tes/Repository/PostgreSqlCachingRepository.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Polly; @@ -16,6 +17,7 @@ namespace Tes.Repository { public abstract class PostgreSqlCachingRepository : IDisposable where T : class { + private readonly IServiceScopeFactory _scopeFactory = null!; private readonly TimeSpan _writerWaitTime = TimeSpan.FromMilliseconds(50); private readonly int _batchSize = 1000; private static readonly TimeSpan defaultCompletedTaskCacheExpiration = TimeSpan.FromDays(1); @@ -30,17 +32,16 @@ public abstract class PostgreSqlCachingRepository : IDisposable where T : cla private readonly Task _writerWorkerTask; protected enum WriteAction { Add, Update, Delete } - - protected Func CreateDbContext { get; init; } protected readonly ICache _cache; protected readonly ILogger _logger; private bool _disposedValue; - protected PostgreSqlCachingRepository(ILogger logger = default, ICache cache = default) + protected PostgreSqlCachingRepository(ILogger logger = default, ICache cache = default, IServiceScopeFactory scopeFactory = default) { _logger = logger; _cache = cache; + _scopeFactory = scopeFactory; // The only "normal" exit for _writerWorkerTask is "cancelled". Anything else should force the process to exit because it means that this repository will no longer write to the database! _writerWorkerTask = Task.Run(() => WriterWorkerAsync(_writerWorkerCancellationTokenSource.Token)) @@ -187,7 +188,8 @@ private async ValueTask WriteItemsAsync(IList<(T DbItem, WriteAction Action, Tas if (dbItems.Count == 0) { return; } cancellationToken.ThrowIfCancellationRequested(); - using var dbContext = CreateDbContext(); + using var scope = _scopeFactory.CreateScope(); + using var dbContext = scope.ServiceProvider.GetRequiredService(); // Manually set entity state to avoid potential NPG PostgreSql bug dbContext.ChangeTracker.AutoDetectChangesEnabled = false; diff --git a/src/Tes/Repository/TesDbContext.cs b/src/Tes/Repository/TesDbContext.cs index 92aab4caf..1f288fcca 100644 --- a/src/Tes/Repository/TesDbContext.cs +++ b/src/Tes/Repository/TesDbContext.cs @@ -1,68 +1,39 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; +using Azure.Core; using Azure.Identity; using Microsoft.EntityFrameworkCore; using Tes.Models; +using Tes.Utilities; namespace Tes.Repository { public class TesDbContext : DbContext { private const int maxBatchSize = 1000; - private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; - public const string TesTasksPostgresTableName = "testasks"; - public bool UseManagedIdentity { get; set; } + private readonly PostgresConnectionStringUtility connectionStringUtility = null!; public TesDbContext() { // Default constructor, which is required to run the EF migrations tool, // "dotnet ef migrations add InitialCreate" + // DI will NOT use this constructor } - public TesDbContext(string connectionString, bool useManagedIdentity = false) + public TesDbContext(PostgresConnectionStringUtility connectionStringUtility) { - ArgumentException.ThrowIfNullOrEmpty(connectionString, nameof(connectionString)); - ConnectionString = connectionString; - UseManagedIdentity = useManagedIdentity; + this.connectionStringUtility = connectionStringUtility; } - public string ConnectionString { get; set; } public DbSet TesTasks { get; set; } - protected override async void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - string tempConnectionString = ConnectionString; - - if (UseManagedIdentity) - { - // Use AAD managed identity - // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity - // https://learn.microsoft.com/en-us/azure/postgresql/single-server/concepts-azure-ad-authentication - // Instructions: - // 1. Replace 'myuser' and run on your server - /* - SET aad_validate_oids_in_tenant = off; - CREATE ROLE myuser WITH LOGIN PASSWORD 'CLIENT_ID' IN ROLE azure_ad_user; - */ - // 2. Set "PostgreSql.UseManagedIdentity" to "true" in the TES AKS configuration - // 3. Ensure the managed identity that TES runs under has the role - - // Note: this supports token caching internally - var credential = new DefaultAzureCredential(); - var accessToken = await credential.GetTokenAsync( - new Azure.Core.TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope })); - - // ConnectionStringUtility omits password when UseManagedIdentity is set, therefore - // omitting this to avoid performance hit of string comparison on every creation - // if (tempConnectionString.Contains("Password=", StringComparison.OrdinalIgnoreCase)) throw new Exception("Password shall not be provided when using managed identity"); - - tempConnectionString = $"{tempConnectionString.TrimEnd(';')};Password={accessToken.Token};"; - } + string connectionString = this.connectionStringUtility.GetConnectionString().Result; optionsBuilder - .UseNpgsql(tempConnectionString, options => options.MaxBatchSize(maxBatchSize)) + .UseNpgsql(connectionString, options => options.MaxBatchSize(maxBatchSize)) .UseLowerCaseNamingConvention(); } } diff --git a/src/Tes/Repository/TesTaskPostgreSqlRepository.cs b/src/Tes/Repository/TesTaskPostgreSqlRepository.cs index fc25cdd2d..de3f01be3 100644 --- a/src/Tes/Repository/TesTaskPostgreSqlRepository.cs +++ b/src/Tes/Repository/TesTaskPostgreSqlRepository.cs @@ -11,8 +11,8 @@ namespace Tes.Repository using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; - using Microsoft.Extensions.Options; using Polly; using Tes.Models; using Tes.Utilities; @@ -23,34 +23,24 @@ namespace Tes.Repository /// public sealed class TesTaskPostgreSqlRepository : PostgreSqlCachingRepository, IRepository { + private readonly IServiceScopeFactory _scopeFactory = null!; + /// /// Default constructor that also will create the schema if it does not exist /// /// /// /// - public TesTaskPostgreSqlRepository(IOptions options, ILogger logger, ICache cache = null) + public TesTaskPostgreSqlRepository(ILogger logger = default, IServiceScopeFactory scopeFactory = default, ICache cache = null) : base(logger, cache) { - var connectionString = new ConnectionStringUtility().GetPostgresConnectionString(options); - CreateDbContext = () => { return new TesDbContext(connectionString, options.Value.UseManagedIdentity); }; - using var dbContext = CreateDbContext(); + _scopeFactory = scopeFactory; + using var scope = _scopeFactory.CreateScope(); + using var dbContext = scope.ServiceProvider.GetRequiredService(); dbContext.Database.MigrateAsync().Wait(); WarmCacheAsync(CancellationToken.None).Wait(); } - /// - /// Constructor for testing to enable mocking DbContext - /// - /// A delegate that creates a TesTaskPostgreSqlRepository context - public TesTaskPostgreSqlRepository(Func createDbContext) - : base() - { - CreateDbContext = createDbContext; - using var dbContext = createDbContext(); - dbContext.Database.MigrateAsync().Wait(); - } - private async Task WarmCacheAsync(CancellationToken cancellationToken) { if (_cache is null) @@ -224,7 +214,8 @@ private async Task GetItemFromCacheOrDatabase(string id, bo if (!_cache?.TryGetValue(id, out item) ?? true) { - using var dbContext = CreateDbContext(); + using var scope = _scopeFactory.CreateScope(); + using var dbContext = scope.ServiceProvider.GetRequiredService(); // Search for Id within the JSON item = await _asyncPolicy.ExecuteAsync(ct => dbContext.TesTasks.FirstOrDefaultAsync(t => t.Json.Id == id, ct), cancellationToken); @@ -252,7 +243,8 @@ private async Task> InternalGetItemsAsync(Expression q.OrderBy(t => t.Json.CreationTime).ThenBy(t => t.Json.Id); orderBy = pagination is null ? orderBy : q => q.OrderBy(t => t.Json.Id); - using var dbContext = CreateDbContext(); + using var scope = _scopeFactory.CreateScope(); + using var dbContext = scope.ServiceProvider.GetRequiredService(); return (await GetItemsAsync(dbContext.TesTasks, WhereTesTask(predicate), cancellationToken, orderBy, pagination)).Select(item => EnsureActiveItemInCache(item, t => t.Json.Id, t => t.Json.IsActiveState()).Json); } diff --git a/src/Tes/Utilities/PostgresConnectionStringUtility.cs b/src/Tes/Utilities/PostgresConnectionStringUtility.cs index 96e1ae6f3..25476141e 100644 --- a/src/Tes/Utilities/PostgresConnectionStringUtility.cs +++ b/src/Tes/Utilities/PostgresConnectionStringUtility.cs @@ -3,51 +3,81 @@ using System; using System.Text; -using Microsoft.Extensions.Options; +using System.Threading.Tasks; +using Azure.Core; using Tes.Models; namespace Tes.Utilities { - public class ConnectionStringUtility + public class PostgresConnectionStringUtility { - public string GetPostgresConnectionString(IOptions options) + private const string azureDatabaseForPostgresqlScope = "https://ossrdbms-aad.database.windows.net/.default"; + private readonly string connectionString = null!; + private readonly TokenCredential tokenCredential = null!; + public bool UseManagedIdentity { get; set; } + + public PostgresConnectionStringUtility(PostgreSqlOptions options, TokenCredential tokenCredential) + { + this.tokenCredential = tokenCredential; + connectionString = InternalGetConnectionString(options); + UseManagedIdentity = options.UseManagedIdentity; + } + + public async Task GetConnectionString() { - ArgumentException.ThrowIfNullOrEmpty(options.Value.ServerName, nameof(options.Value.ServerName)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.ServerNameSuffix, nameof(options.Value.ServerNameSuffix)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.ServerPort, nameof(options.Value.ServerPort)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.ServerSslMode, nameof(options.Value.ServerSslMode)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseName, nameof(options.Value.DatabaseName)); - ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserLogin, nameof(options.Value.DatabaseUserLogin)); - - if (!options.Value.UseManagedIdentity) + if (UseManagedIdentity) + { + // Use AAD managed identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-connect-with-managed-identity + // https://learn.microsoft.com/en-us/azure/postgresql/single-server/concepts-azure-ad-authentication + + var accessToken = await tokenCredential.GetTokenAsync( + new TokenRequestContext(scopes: new string[] { azureDatabaseForPostgresqlScope }), System.Threading.CancellationToken.None); + + return $"{connectionString}Password={accessToken.Token};"; + } + + return connectionString; + } + + private string InternalGetConnectionString(PostgreSqlOptions options) + { + ArgumentException.ThrowIfNullOrEmpty(options.ServerName, nameof(options.ServerName)); + ArgumentException.ThrowIfNullOrEmpty(options.ServerNameSuffix, nameof(options.ServerNameSuffix)); + ArgumentException.ThrowIfNullOrEmpty(options.ServerPort, nameof(options.ServerPort)); + ArgumentException.ThrowIfNullOrEmpty(options.ServerSslMode, nameof(options.ServerSslMode)); + ArgumentException.ThrowIfNullOrEmpty(options.DatabaseName, nameof(options.DatabaseName)); + ArgumentException.ThrowIfNullOrEmpty(options.DatabaseUserLogin, nameof(options.DatabaseUserLogin)); + + if (!options.UseManagedIdentity) { // Ensure password is set if NOT using Managed Identity - ArgumentException.ThrowIfNullOrEmpty(options.Value.DatabaseUserPassword, nameof(options.Value.DatabaseUserPassword)); + ArgumentException.ThrowIfNullOrEmpty(options.DatabaseUserPassword, nameof(options.DatabaseUserPassword)); } - if (options.Value.UseManagedIdentity && !string.IsNullOrWhiteSpace(options.Value.DatabaseUserPassword)) + if (options.UseManagedIdentity && !string.IsNullOrWhiteSpace(options.DatabaseUserPassword)) { // throw if password IS set when using Managed Identity throw new ArgumentException("DatabaseUserPassword shall not be set if UseManagedIdentity is true"); } - if (options.Value.ServerName.Contains(options.Value.ServerNameSuffix, StringComparison.OrdinalIgnoreCase)) + if (options.ServerName.Contains(options.ServerNameSuffix, StringComparison.OrdinalIgnoreCase)) { - throw new ArgumentException($"'{nameof(options.Value.ServerName)}' should only contain the name of the server like 'myserver' and NOT the full host name like 'myserver{options.Value.ServerNameSuffix}'", nameof(options.Value.ServerName)); + throw new ArgumentException($"'{nameof(options.ServerName)}' should only contain the name of the server like 'myserver' and NOT the full host name like 'myserver{options.ServerNameSuffix}'", nameof(options.ServerName)); } var connectionStringBuilder = new StringBuilder(); - connectionStringBuilder.Append($"Server={options.Value.ServerName}{options.Value.ServerNameSuffix};"); - connectionStringBuilder.Append($"Database={options.Value.DatabaseName};"); - connectionStringBuilder.Append($"Port={options.Value.ServerPort};"); - connectionStringBuilder.Append($"User Id={options.Value.DatabaseUserLogin};"); + connectionStringBuilder.Append($"Server={options.ServerName}{options.ServerNameSuffix};"); + connectionStringBuilder.Append($"Database={options.DatabaseName};"); + connectionStringBuilder.Append($"Port={options.ServerPort};"); + connectionStringBuilder.Append($"SSL Mode={options.ServerSslMode};"); + connectionStringBuilder.Append($"User Id={options.DatabaseUserLogin};"); - if (!options.Value.UseManagedIdentity) + if (!options.UseManagedIdentity) { - connectionStringBuilder.Append($"Password={options.Value.DatabaseUserPassword};"); + connectionStringBuilder.Append($"Password={options.DatabaseUserPassword};"); } - connectionStringBuilder.Append($"SSL Mode={options.Value.ServerSslMode};"); return connectionStringBuilder.ToString(); } } diff --git a/src/TesApi.Tests/Repository/TesTaskPostgreSqlRepositoryIntegrationTests.cs b/src/TesApi.Tests/Repository/TesTaskPostgreSqlRepositoryIntegrationTests.cs index ca192e7e6..99058d260 100644 --- a/src/TesApi.Tests/Repository/TesTaskPostgreSqlRepositoryIntegrationTests.cs +++ b/src/TesApi.Tests/Repository/TesTaskPostgreSqlRepositoryIntegrationTests.cs @@ -61,10 +61,9 @@ await PostgreSqlTestUtility.CreateTestDbAsync( DatabaseUserPassword = adminPw }; - var optionsMock = new Mock>(); - optionsMock.Setup(x => x.Value).Returns(options); - var connectionString = new ConnectionStringUtility().GetPostgresConnectionString(optionsMock.Object); - repository = new TesTaskPostgreSqlRepository(() => new TesDbContext(connectionString)); + var optionsMock = new Mock(); + var connectionString = new PostgresConnectionStringUtility(optionsMock.Object, null); + repository = new TesTaskPostgreSqlRepository(); Console.WriteLine("Creation complete."); } diff --git a/src/TesApi.Web/Startup.cs b/src/TesApi.Web/Startup.cs index e28d2c4d2..5463f7e78 100644 --- a/src/TesApi.Web/Startup.cs +++ b/src/TesApi.Web/Startup.cs @@ -20,6 +20,7 @@ using Tes.ApiClients.Options; using Tes.Models; using Tes.Repository; +using Tes.Utilities; using TesApi.Filters; using TesApi.Web.Management; using TesApi.Web.Management.Batch; @@ -77,6 +78,10 @@ public void ConfigureServices(IServiceCollection services) .Configure(configuration.GetSection(MarthaOptions.SectionName)) .AddMemoryCache(o => o.ExpirationScanFrequency = TimeSpan.FromHours(12)) + + .AddSingleton() + .AddSingleton() + .AddDbContext(ServiceLifetime.Scoped) .AddSingleton, TesRepositoryCache>() .AddSingleton() .AddSingleton() @@ -108,7 +113,6 @@ public void ConfigureServices(IServiceCollection services) .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(s => new DefaultAzureCredential()) .AddSingleton() .AddSingleton() .AddTransient() From f956900a620e998c63d5e724de83234bc343947f Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 10 Oct 2023 16:28:25 -0700 Subject: [PATCH 19/19] update nuget --- src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj | 2 +- src/Tes.ApiClients/Tes.ApiClients.csproj | 2 +- src/Tes.Runner.Test/Tes.Runner.Test.csproj | 2 +- src/Tes.Runner/Tes.Runner.csproj | 2 +- src/Tes/Tes.csproj | 12 ++++++------ src/TesApi.Web/TesApi.Web.csproj | 2 +- src/deploy-tes-on-azure/deploy-tes-on-azure.csproj | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj b/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj index 6c38ebbb8..7f2257ba3 100644 --- a/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj +++ b/src/GenerateBatchVmSkus/GenerateBatchVmSkus.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Tes.ApiClients/Tes.ApiClients.csproj b/src/Tes.ApiClients/Tes.ApiClients.csproj index 446559573..9e3752919 100644 --- a/src/Tes.ApiClients/Tes.ApiClients.csproj +++ b/src/Tes.ApiClients/Tes.ApiClients.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Tes.Runner.Test/Tes.Runner.Test.csproj b/src/Tes.Runner.Test/Tes.Runner.Test.csproj index 6c3a99410..49711d9fe 100644 --- a/src/Tes.Runner.Test/Tes.Runner.Test.csproj +++ b/src/Tes.Runner.Test/Tes.Runner.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Tes.Runner/Tes.Runner.csproj b/src/Tes.Runner/Tes.Runner.csproj index 35f75644d..73ea6d965 100644 --- a/src/Tes.Runner/Tes.Runner.csproj +++ b/src/Tes.Runner/Tes.Runner.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/Tes/Tes.csproj b/src/Tes/Tes.csproj index 2448667de..e75c31818 100644 --- a/src/Tes/Tes.csproj +++ b/src/Tes/Tes.csproj @@ -5,20 +5,20 @@ - + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + - + diff --git a/src/TesApi.Web/TesApi.Web.csproj b/src/TesApi.Web/TesApi.Web.csproj index ea889c7b5..5697eeff8 100644 --- a/src/TesApi.Web/TesApi.Web.csproj +++ b/src/TesApi.Web/TesApi.Web.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj b/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj index 6c662bbee..373b43bb0 100644 --- a/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj +++ b/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj @@ -26,7 +26,7 @@ - +