Merge pull request #376 from cmu-sei/v8

continued refinement of chat api

Author: sei-dupdyke

README.md

@@ -4,6 +4,8 @@ GHOSTS simulates what anyone might do at a computer, creating documents, browsin
 
 GHOSTS has many use cases in cyber training and exercises, most notably for bringing non-player characters (NPCs) to life, but GHOSTS can be used for many other purposes where realistic activity on a computer is needed as well.
 
+There is a [short demonstration video available on YouTube](https://www.youtube.com/watch?v=EkwK-cqwjjA) (3:03).
+
 ---
 
 **Version 8 is here (with breaking changes!).** It has absorbed the other modules of the GHOSTS framework, [ANIMATOR (now archived)](https://github.com/cmu-sei/GHOSTS-ANIMATOR) and [SPECTRE (now archived)](https://github.com/cmu-sei/GHOSTS-SPECTRE). This was done in order to greatly simplify installation, configuration, and the administration of a GHOSTS instance, but also to bring further capability to the core agents by more tightly combining information segregated into separate databases and systems until now.
@@ -18,8 +20,6 @@ Sorry, but there is no upgrade path from previous versions — install a fresh i
 
 ---
 
-There is a [short demonstration video available on YouTube](https://www.youtube.com/watch?v=EkwK-cqwjjA) (3:03).
-
 ## Key Links
 
 - [Quick Start: Installation from distribution binaries](https://cmu-sei.github.io/GHOSTS/quickstart/)
@@ -46,6 +46,18 @@ The API server provides a way for clients to interact with the GHOSTS system and
 - Get/manage information from clients regarding their previous or current activities, etc.
 - Orchestrate new activities for particular clients to perform
 
+### [Ghosts Lite](src/Ghosts.Client.Lite/)
+
+A resource light version of the Windows GHOSTS client that can be run on minimal hardware.
+
+### [Pandora Content Server](src/ghosts.pandora/)
+
+A server that provides content to GHOSTS clients (or otherwise). Pandora determines what you most likely requested, creates that content, and serves it back in the response. Pandora also has the ability to serve predetermined static content for training and exercise purposes (and red-teaming).
+
+### [Pandora Socializer Server](src/ghosts.pandora.socializer/)
+
+The social media (x.com) server that enables Ghosts clients to post and interact with social media content.
+
 ## License
 
 [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution.

src/Ghosts.Api/Infrastructure/Animations/AnimationDefinitions/Chat/ChatClient.cs

@@ -10,8 +10,8 @@
 using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
-using Ghosts.Api;
 using ghosts.api.Hubs;
+using Ghosts.Api.Infrastructure;
 using ghosts.api.Infrastructure.Animations.AnimationDefinitions.Chat.Mattermost;
 using ghosts.api.Infrastructure.ContentServices;
 using Ghosts.Api.Infrastructure.Data;
@@ -28,6 +28,7 @@ public class ChatClient
 {
     private static readonly Logger _log = LogManager.GetCurrentClassLogger();
     private readonly ChatJobConfiguration _configuration;
+    private readonly ApplicationSettings.AnimatorSettingsDetail.AnimationsSettings.ChatSettings _chatSettings;
     private readonly string _baseUrl;
     private readonly HttpClient _client;
     private string _token;
@@ -37,9 +38,10 @@ public class ChatClient
     private readonly ApplicationDbContext _context;
     private IHubContext<ActivityHub> _activityHubContext;
 
-    public ChatClient(ChatJobConfiguration config, IFormatterService formatterService, IHubContext<ActivityHub> activityHubContext, ApplicationDbContext context)
+    public ChatClient(ApplicationSettings.AnimatorSettingsDetail.AnimationsSettings.ChatSettings chatSettings, ChatJobConfiguration config, IFormatterService formatterService, IHubContext<ActivityHub> activityHubContext, ApplicationDbContext context)
     {
         _configuration = config;
+        _chatSettings = chatSettings;
         this._baseUrl = _configuration.Chat.BaseUrl;
         this._client = new HttpClient();
         this._formatterService = formatterService;
@@ -88,8 +90,8 @@ private async Task<User> Login(string username, string password)
         }
         catch (Exception e)
         {
-            _log.Error(e);
-            return null;
+            _log.Error($"Cannot login {username}:{password} with error {e.Message}|{e.StackTrace}");
+            throw;
         }
     }
 
@@ -130,7 +132,7 @@ private async Task<IEnumerable<Channel>> GetMyChannels(User user)
         }
         catch (Exception e)
         {
-            _log.Error($"No channels found {e}");
+            _log.Trace($"No channels found {e}");
             return new List<Channel>();
         }
     }
@@ -503,32 +505,49 @@ private async Task StepEx(Random random, Guid NpcId, string username, string pas
                         if (lastPost != null)
                             postId = lastPost.PostId;
                         var posts = await this.GetPostsByChannel(channel.Id, postId);
-                        foreach (var post in posts.Posts.Where(x => x.Value.Type == ""))
+                        try
                         {
-                            var user = await this.GetUserById(post.Value.UserId);
-                            if (user == null)
+                            if (posts?.Posts != null)
                             {
-                                const string email = "[email protected]";
-                                user = new User
-                                    { FirstName = "some", LastName = "one", Email = email, Username = email.CreateUsernameFromEmail() };
+                                foreach (var post in posts.Posts.Where(x => x.Value?.Type == ""))
+                                {
+                                    var user = await this.GetUserById(post.Value.UserId);
+                                    if (user == null)
+                                    {
+                                        const string email = "[email protected]";
+                                        user = new User
+                                        {
+                                            FirstName = "some",
+                                            LastName = "one",
+                                            Email = email,
+                                            Username = email.CreateUsernameFromEmail()
+                                        };
+                                    }
+
+                                    channelHistory.Add(new ChannelHistory
+                                    {
+                                        ChannelId = channel.Id,
+                                        ChannelName = channel.Name,
+                                        UserId = post.Value.Id,
+                                        PostId = post.Value.Id,
+                                        UserName = user.Username,
+                                        Created = post.Value.CreateAt.ToDateTime(),
+                                        Message = post.Value.Message
+                                    });
+                                }
                             }
-
-                            channelHistory.Add(new ChannelHistory
-                            {
-                                ChannelId = channel.Id, ChannelName = channel.Name, UserId = post.Value.Id,
-                                PostId = post.Value.Id, 
-                                UserName = user.Username,
-                                Created = post.Value.CreateAt.ToDateTime(),
-                                Message = post.Value.Message
-                            });
+                        }
+                        catch (Exception ex)
+                        {
+                            _log.Trace($"An error occurred: {ex.Message}");
                         }
                     }
                 }
 
                 var subPrompts = this._configuration.Prompts.GetRandom(random);
                 _log.Trace($"{username} looking at posts...");
 
-                if (random.Next(0, 100) > 50)
+                if (random.Next(0, 99) < _chatSettings.PercentWillPost)
                 {
                     _log.Info($"{username} exiting.");
                     return;
@@ -550,11 +569,13 @@ private async Task StepEx(Random random, Guid NpcId, string username, string pas
 
                 var history = channelHistory.Where(x => x.ChannelId == randomChannelToPostTo && x.UserName != me.Username).MaxBy(x => x.Created);
                 //var historyString = history is { Message.Length: >= 100 } ? history.Message[..100] : history?.Message;
-                var historyString = history.Message;
+                var historyString = string.Empty;
+                if (history != null)
+                    historyString = history.Message;
 
                 var prompt = $"Write my update to the chat system that {subPrompts}";
                 var respondingTo = string.Empty;
-                if (random.Next(0, 99) < Program.ApplicationSettings.AnimatorSettings.Animations.Chat.PercentReplyVsNew && !string.IsNullOrEmpty(historyString) && history.UserId != me.Id)
+                if (random.Next(0, 99) < _chatSettings.PercentReplyVsNew && !string.IsNullOrEmpty(historyString) && history.UserId != me.Id)
                 {
                     prompt =
                         $"How do I respond to this? {historyString}";

src/Ghosts.Api/Infrastructure/Animations/AnimationDefinitions/ChatJob.cs

@@ -20,15 +20,15 @@ namespace ghosts.api.Infrastructure.Animations.AnimationDefinitions;
 public class ChatJob
 {
     private static readonly Logger _log = LogManager.GetCurrentClassLogger();
-    private readonly ApplicationSettings _configuration;
+    private readonly ApplicationSettings.AnimatorSettingsDetail.AnimationsSettings.ChatSettings _configuration;
     private readonly ApplicationDbContext _context;
     private readonly Random _random;
     private readonly ChatClient _chatClient;
     private readonly int _currentStep;
     private CancellationToken _cancellationToken;
     private IFormatterService _formatterService;
 
-    public ChatJob(ApplicationSettings configuration, IServiceScopeFactory scopeFactory, Random random,
+    public ChatJob(ApplicationSettings.AnimatorSettingsDetail.AnimationsSettings.ChatSettings configuration, IServiceScopeFactory scopeFactory, Random random,
         IHubContext<ActivityHub> activityHubContext, CancellationToken cancellationToken)
     {
         //todo: post results to activityHubContext for "top" reporting
@@ -45,20 +45,20 @@ public ChatJob(ApplicationSettings configuration, IServiceScopeFactory scopeFact
             new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new InvalidOperationException();
 
         this._formatterService =
-            new ContentCreationService(_configuration.AnimatorSettings.Animations.Chat.ContentEngine).FormatterService;
+            new ContentCreationService(_configuration.ContentEngine).FormatterService;
 
-        this._chatClient = new ChatClient(chatConfiguration, this._formatterService, activityHubContext, this._context);
+        this._chatClient = new ChatClient(_configuration, chatConfiguration, this._formatterService, activityHubContext, this._context);
 
         while (!_cancellationToken.IsCancellationRequested)
         {
-            if (this._currentStep > _configuration.AnimatorSettings.Animations.Chat.MaximumSteps)
+            if (this._currentStep > _configuration.MaximumSteps)
             {
                 _log.Trace($"Maximum steps met: {this._currentStep - 1}. Chat Job is exiting...");
                 return;
             }
 
             this.Step(random, chatConfiguration);
-            Thread.Sleep(this._configuration.AnimatorSettings.Animations.Chat.TurnLength);
+            Thread.Sleep(this._configuration.TurnLength);
 
             this._currentStep++;
         }

src/Ghosts.Api/Infrastructure/Animations/AnimationsManager.cs

@@ -319,13 +319,13 @@ private void Run(AnimationConfiguration animationConfiguration)
                     _chatJobThread = new Thread(() =>
                     {
                         Thread.CurrentThread.IsBackground = true;
-                        _ = new ChatJob(settings, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
+                        _ = new ChatJob(chatSettings, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
                     });
                     _chatJobThread.Start();
                 }
                 else
                 {
-                    _ = new ChatJob(settings, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
+                    _ = new ChatJob(chatSettings, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
                 }
 
                 break;
@@ -435,13 +435,13 @@ private void Run()
                     _chatJobThread = new Thread(() =>
                     {
                         Thread.CurrentThread.IsBackground = true;
-                        _ = new ChatJob(this._configuration, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
+                        _ = new ChatJob(this._configuration.AnimatorSettings.Animations.Chat, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
                     });
                     _chatJobThread.Start();
                 }
                 else
                 {
-                    _ = new ChatJob(this._configuration, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
+                    _ = new ChatJob(this._configuration.AnimatorSettings.Animations.Chat, _scopeFactory, this._random, this._activityHubContext, this._chatJobJobCancellationTokenSource.Token);
                 }
             }
             else

src/Ghosts.Api/Infrastructure/ApplicationSettings.cs

@@ -82,6 +82,7 @@ public class ChatSettings
                 public int MaximumSteps { get; set; }
                 public bool IsSendingTimelinesToGhostsApi { get; set; }
                 public int PercentReplyVsNew { get; set; }
+                public int PercentWillPost { get; set; }
                 public string PostUrl { get; set; }
                 public ContentEngineSettings ContentEngine { get; set; }
             }