Running BFF on multiple pods

[mmingle]

Which version of Duende BFF are you using?
2.2.0
Which version of .NET are you using?
6
Describe the bug

This more of a question, but is there any known issues or docs for running a bff service on multiple pods? I am trying to use redis for session management. I implemented a custom user store and added it to the startup

services.AddBff(options => { options.EnforceBffMiddleware = true; options.LicenseKey = cpConfig.License; options.AnonymousSessionResponse = AnonymousSessionResponse.Response200; }) .AddRemoteApis() .AddServerSideSessions<RedisUserSessionStore>();
All of that seems to work great when there is only 1 instance, but as soon as I push to upper environments I get the following issue
Access token is missing. token type: '"Unknown token type"', local path: '"Unknown Route"', detail: '"Missing access token"'

[mmingle]

will I also need to implement something for the token storage?

[mmingle]

I figured it out! For anyone stumbling across this in the future, the issue was around data protection keys. Each pod was using a different key. I needed to add something like this

services.AddDataProtection()
.PersistKeysToStackExchangeRedis(ConnectionMultiplexer.Connect("redis connection string"), "DataProtection-Keys")
.SetApplicationName("MyApp");

[mmingle]

also for future people here is the rough datastore I created using redis. (still needs some work, bug it is a starting point)

  public class RedisUserSessionStore : IUserSessionStore, IUserSessionStoreCleanup
 {
     private readonly IConnectionMultiplexer _redis;
     private readonly ILogger<RedisUserSessionStore> _logger;
     private readonly IDatabase _database;
     private const string KeyPrefix = "YourApp";

     public RedisUserSessionStore(IConnectionMultiplexer redis, ILogger<RedisUserSessionStore> logger)
     {
         _redis = redis;
         _database = _redis.GetDatabase();
         _logger = logger;
     }

     public async Task<UserSession?> GetUserSessionAsync(string key, CancellationToken cancellationToken = default)
     {
         if (!key.Contains(KeyPrefix))
         {
             key = $"{KeyPrefix}-{key}";
         }
         var sessionData = await _database.StringGetAsync(key);
         if (sessionData.IsNullOrEmpty)
         {
             _logger.LogInformation($"GetUserSessionAsync for key {key} was null");
             return null;
         }
         return JsonSerializer.Deserialize<UserSession>(sessionData);
     }

     public async Task CreateUserSessionAsync(UserSession session, CancellationToken cancellationToken = default)
     {
         session.Key = $"{KeyPrefix}-{session.Key}";
         var expires = TimeSpan.FromDays(1);
         if (session.Expires.HasValue)
         {
             expires = session.Expires.Value - DateTime.UtcNow;
         }

         var sessionData = JsonSerializer.Serialize(session);
         await _database.StringSetAsync(session.Key, sessionData, expires, When.Always, CommandFlags.None);

         // Add to indexes
         await _database.SetAddAsync($"{KeyPrefix}-SubjectId:{session.SubjectId}", session.Key);
         await _database.SetAddAsync($"{KeyPrefix}-SessionId:{session.SessionId}", session.Key);
     }

     public async Task UpdateUserSessionAsync(string key, UserSessionUpdate sessionUpdate, CancellationToken cancellationToken = default)
     {
         key = $"{KeyPrefix}-{key}";
         var expires = TimeSpan.FromDays(1);
         if (sessionUpdate.Expires.HasValue)
         {
             expires = sessionUpdate.Expires.Value - DateTime.UtcNow;
         }

         var session = await GetUserSessionAsync(key, cancellationToken);
         if (session == null)
         {
             throw new KeyNotFoundException($"No session found with key: {key}");
         }
         var sessionData = JsonSerializer.Serialize(sessionUpdate);
         await _database.StringSetAsync(key, sessionData, expires, When.Always, CommandFlags.None);
     }

     public async Task DeleteUserSessionAsync(string key, CancellationToken cancellationToken = default)
     {
         var session = await GetUserSessionAsync(key, cancellationToken);
         if (session != null)
         {
             await _database.KeyDeleteAsync(key);

             // Remove from indexes
             await _database.SetRemoveAsync($"{KeyPrefix}-SubjectId:{session.SubjectId}", key);
             await _database.SetRemoveAsync($"{KeyPrefix}-SessionId:{session.SessionId}", key);
         }
     }

     public async Task<IReadOnlyCollection<UserSession>> GetUserSessionsAsync(UserSessionsFilter filter, CancellationToken cancellationToken = default)
     {
         var sessions = new List<UserSession>();

         // Filtering by SubjectId and SessionId
         var keys = new HashSet<RedisValue>();

         if (!string.IsNullOrEmpty(filter.SubjectId))
         {
             var subjectKeys = await _database.SetMembersAsync($"{KeyPrefix}-SubjectId:{filter.SubjectId}");
             foreach (var key in subjectKeys)
             {
                 keys.Add(key);
             }
         }

         if (!string.IsNullOrEmpty(filter.SessionId))
         {
             var sessionKeys = await _database.SetMembersAsync($"{KeyPrefix}-SessionId:{filter.SessionId}");
             foreach (var key in sessionKeys)
             {
                 keys.Add(key);
             }
         }

         foreach (var key in keys)
         {
             var session = await GetUserSessionAsync(key, cancellationToken);
             if (session != null)
             {
                 sessions.Add(session);
             }
         }

         return sessions;
     }

     public async Task DeleteUserSessionsAsync(UserSessionsFilter filter, CancellationToken cancellationToken = default)
     {
         filter.Validate();
         var keys = new HashSet<RedisValue>();

         if (!string.IsNullOrEmpty(filter.SubjectId))
         {
             var subjectKeys = await _database.SetMembersAsync($"{KeyPrefix}-SubjectId:{filter.SubjectId}");
             foreach (var key in subjectKeys)
             {
                 keys.Add(key);
             }
         }

         if (!string.IsNullOrEmpty(filter.SessionId))
         {
             var sessionKeys = await _database.SetMembersAsync($"{KeyPrefix}-SessionId:{filter.SessionId}");
             foreach (var key in sessionKeys)
             {
                 keys.Add(key);
             }
         }

         foreach (var key in keys)
         {
             await DeleteUserSessionAsync(key, cancellationToken);
         }
     }

     public Task DeleteExpiredSessionsAsync(CancellationToken cancellationToken = default)
     {
         //TODO: Implement this.
         throw new NotImplementedException();
     }
 }