graph: allow graphs to be auto-collected (controlled via yb_graph_auto_collect_db)

By default it's off, but it's allow bot to scan graph directory, do a diff with a graph db server and upload every single graph file that do not exist in central database. This is done in a separate thread and do not block server process, and only once server/game is started, not on change level. Also, it's not working on currently started map.

cfg/addons/yapb/conf/yapb.cfg

@@ -38,7 +38,7 @@ yb_graph_analyze_max_jump_height "44"
 //
 // The FPS at which analyzer process is running. This keeps game from freezing during analyzing.
 // ---
-// Default: "30.0"
+// Default: "30.0", Min: "25.0", Max: "99.0"
 // 
 yb_graph_analyze_fps "30.0"
 
@@ -345,6 +345,13 @@ yb_ignore_map_prefix_game_mode "0"
 // 
 yb_threadpool_workers "-1"
 
+//
+// If enabled, bots will not apply throwing condition on grenades.
+// ---
+// Default: "0", Min: "0", Max: "1"
+// 
+yb_grenadier_mode "0"
+
 //
 // Specifies whether bot should not 'fix' camp directions of camp waypoints when loading old PWF format.
 // ---
@@ -380,6 +387,13 @@ yb_graph_auto_save_count "15"
 // 
 yb_graph_draw_distance "400"
 
+//
+// Allows bot's to exchange your graph files with graph database automatically.
+// ---
+// Default: "0", Min: "0", Max: "1"
+// 
+yb_graph_auto_collect_db "0"
+
 //
 // Kick bots to automatically make room for human players.
 // ---
@@ -388,7 +402,7 @@ yb_graph_draw_distance "400"
 yb_autovacate "1"
 
 //
-// How many slots autovacate feature should keep for human players
+// How many slots autovacate feature should keep for human players.
 // ---
 // Default: "1", Min: "1", Max: "8"
 // 
@@ -541,6 +555,15 @@ yb_botskin_t "0"
 // 
 yb_botskin_ct "0"
 
+//
+// Sets the default personality when creating bots with quota management.
+//  Allowed values: 'none', 'normal', 'careful', 'rusher'.
+//  If 'none' is specified personality chosen randomly.
+// ---
+// Default: "none"
+// 
+yb_preferred_personality "none"
+
 //
 // Lower bound for base bot ping shown in scoreboard upon creation.
 // ---
@@ -597,12 +620,19 @@ yb_rotate_stay_min "360.0"
 // 
 yb_rotate_stay_max "3600.0"
 
+//
+// When enabled, bots will not try to avoid teammates on their way. Assuming some of the semiclip plugins are in use.
+// ---
+// Default: "0", Min: "0", Max: "1"
+// 
+yb_has_team_semiclip "0"
+
 //
 // Selects the heuristic function mode. For debug purposes only.
 // ---
-// Default: "3", Min: "0", Max: "4"
+// Default: "0", Min: "0", Max: "4"
 // 
-yb_path_heuristic_mode "3"
+yb_path_heuristic_mode "0"
 
 //
 // Limit maximum floyd-warshall memory (megabytes). Use Dijkstra if memory exceeds.
@@ -625,6 +655,13 @@ yb_path_dijkstra_simple_distance "1"
 // 
 yb_path_astar_post_smooth "0"
 
+//
+// Randomize pathfinding on each round start.
+// ---
+// Default: "1", Min: "0", Max: "1"
+// 
+yb_path_randomize_on_round_start "1"
+
 //
 // Enables or disables showing welcome message to host entity on game start.
 // ---
@@ -646,6 +683,20 @@ yb_enable_query_hook "0"
 // 
 yb_breakable_health_limit "500.0"
 
+//
+// Allows or disallows bots to return fake steam id.
+// ---
+// Default: "0", Min: "0", Max: "1"
+// 
+yb_enable_fake_steamids "0"
+
+//
+// Count player pings when calculating average ping for bots. If no, some random ping chosen for bots.
+// ---
+// Default: "1", Min: "0", Max: "1"
+// 
+yb_count_players_for_fakeping "1"
+
 //
 // Specifies whether bots able to use 'shift' if they thinks that enemy is near.
 // ---
@@ -688,3 +739,10 @@ yb_random_knife_attacks "1"
 // 
 yb_max_nodes_for_predict "25"
 
+//
+// Enables or disables extra hard difficulty for bots.
+// ---
+// Default: "0", Min: "0", Max: "1"
+// 
+yb_whose_your_daddy "0"
+

ext/crlib

@@ -1 +1 @@
-Subproject commit aa8e1e0eaed2e4b75e6b2a0fff5a68756cddc972
+Subproject commit 5225f857c44ced8e61e65b25892a0c8ab015250c

inc/graph.h

@@ -168,6 +168,7 @@ private:
    bool m_narrowChecked {};
    bool m_silenceMessages {};
    bool m_lightChecked {};
+   bool m_isOnlineCollected {};
 
    Vector m_learnVelocity {};
    Vector m_learnPosition {};
@@ -259,6 +260,8 @@ public:
    void showStats ();
    void showFileInfo ();
    void emitNotify (int32_t sound);
+   void syncCollectOnline ();
+   void collectOnline ();
 
    IntArray getNearestInRadius (float radius, const Vector &origin, int maxCount = -1);
    const IntArray &getNodesInBucket (const Vector &pos);

inc/product.h

@@ -40,6 +40,7 @@ public:
    static constexpr StringRef url { "https://yapb.jeefo.net/" };
    static constexpr StringRef download { "yapb.jeefo.net" };
    static constexpr StringRef upload { "yapb.jeefo.net/upload" };
+   static constexpr StringRef httpScheme { "http" };
    static constexpr StringRef logtag { "YB" };
    static constexpr StringRef dtime { __DATE__ " " __TIME__ };
    static constexpr StringRef date { __DATE__ };

inc/storage.h

@@ -76,7 +76,7 @@ public:
    template <typename ...Args> bool error (bool isGraph, bool isDebug, MemFile &file, const char *fmt, Args &&...args);
 
    // builds the filename to requested filename
-   String buildPath (int32_t type, bool isMemoryLoad = false);
+   String buildPath (int32_t type, bool isMemoryLoad = false, bool withoutMapName = false);
 
    // get's relative path against bot library (bot library should reside in bin dir)
    StringRef getRunningPath ();

src/control.cpp

@@ -843,11 +843,11 @@ int BotControl::cmdNodeUpload () {
    String uploadUrlAddress = cv_graph_url_upload.str ();
 
    // only allow to upload to non-https endpoint
-   if (uploadUrlAddress.startsWith ("http")) {
+   if (uploadUrlAddress.startsWith ("https")) {
       msg ("Value of \"%s\" cvar should not contain URL scheme, only the host name and path.", cv_graph_url_upload.name ());
       return BotCommandResult::Handled;
    }
-   String uploadUrl = strings.format ("http://%s", uploadUrlAddress);
+   String uploadUrl = strings.format ("%s://%s", product.httpScheme, uploadUrlAddress);
 
    msg ("\n");
    msg ("WARNING!");

src/graph.cpp

@@ -12,6 +12,7 @@ ConVar cv_graph_url ("graph_url", product.download.chars (), "Specifies the URL
 ConVar cv_graph_url_upload ("graph_url_upload", product.upload.chars (), "Specifies the URL to which bots will try to upload the graph file to database.", false, 0.0f, 0.0f);
 ConVar cv_graph_auto_save_count ("graph_auto_save_count", "15", "Every N graph nodes placed on map, the graph will be saved automatically (without checks).", true, 0.0f, kMaxNodes);
 ConVar cv_graph_draw_distance ("graph_draw_distance", "400", "Maximum distance to draw graph nodes from editor viewport.", true, 64.0f, 3072.0f);
+ConVar cv_graph_auto_collect_db ("graph_auto_collect_db", "0", "Allows bot's to exchange your graph files with graph database automatically.");
 
 void BotGraph::reset () {
    // this function initialize the graph structures..
@@ -1277,6 +1278,129 @@ void BotGraph::emitNotify (int32_t sound) {
    }
 }
 
+void BotGraph::syncCollectOnline () {
+   m_isOnlineCollected = true; // once per server start
+
+   // path to graph files
+   auto graphFilesPath = bstor.buildPath (BotFile::Graph, false, true);
+
+   // enumerate graph files
+   FileEnumerator enumerator { strings.joinPath (graphFilesPath, "*.graph") };
+
+   // listing of graphs locally available
+   StringArray localGraphs {};
+
+   // collect all the files
+   while (enumerator) {
+      auto match = enumerator.getMatch ();
+
+      match = match.substr (match.findLastOf (kPathSeparator) + 1);
+      match = match.substr (0, match.findFirstOf ("."));
+
+      localGraphs.emplace (match);
+      enumerator.next ();
+   }
+
+   // no graphs ? unbelievable
+   if (localGraphs.empty ()) {
+      return;
+   }
+   String uploadUrlAddress = cv_graph_url_upload.str ();
+
+   // only allow to upload to non-https endpoint
+   if (uploadUrlAddress.startsWith ("https")) {
+      return;
+   }
+   String localFile = plat.tmpfname ();
+
+   // don't forget remove temporary file
+   auto unlinkTemporary = [&] () {
+      if (plat.fileExists (localFile.chars ())) {
+         plat.removeFile (localFile.chars ());
+      }
+   };
+
+   // write out our list of files into temporary
+   if (File lc { localFile, "wt" }) {
+      auto graphList = String::join (localGraphs, ",");
+      auto collectUrl = strings.format ("%s://%s/collect/%u", product.httpScheme, uploadUrlAddress, graphList.hash ());
+
+      lc.puts (graphList.chars ());
+      lc.close ();
+
+      // upload to collection diff
+      if (!http.uploadFile (collectUrl, localFile)) {
+         unlinkTemporary ();
+         return;
+      }
+      unlinkTemporary ();
+
+      // download collection diff
+      if (!http.downloadFile (collectUrl, localFile)) {
+         return;
+      }
+      StringArray wanted {};
+
+      // decode answer
+      if (lc.open (localFile, "rt")) {
+         String lines;
+
+         if (lc.getLine (lines)) {
+            wanted = lines.split (",");
+         }
+         lc.close ();
+      }
+      unlinkTemporary ();
+
+      // if 'we're have something in diff, bailout
+      if (wanted.empty ()) {
+         return;
+      }
+      localGraphs.clear ();
+
+      // convert graphs names into full paths
+      for (const auto &wn : wanted) {
+         if (wn == game.getMapName ()) {
+            continue; // skip current map always
+         }
+         localGraphs.emplace (strings.joinPath (graphFilesPath, wn) + ".graph");
+      }
+
+      // try to upload everything database wants
+      for (const auto &lg : localGraphs) {
+         if (!plat.fileExists (lg.chars ())) {
+            continue;
+         }
+         StorageHeader hdr {};
+
+         // read storage header and check if file NOT analyzed
+         if (File gp { lg, "rb" }) {
+            gp.read (&hdr, sizeof (StorageHeader));
+
+            // check the magic, graph is NOT analyzed and have some viable nodes number
+            if (hdr.magic == kStorageMagic && !(hdr.options & StorageOption::Analyzed) && hdr.length > 48) {
+               String uploadUrl = strings.format ("%s://%s", product.httpScheme, uploadUrlAddress);
+
+               // try to upload to database (no need check if it's succeed)
+               http.uploadFile (uploadUrl, lg);
+            }
+            gp.close ();
+         }
+      }
+   }
+   unlinkTemporary ();
+}
+
+void BotGraph::collectOnline () {
+   if (m_isOnlineCollected || !cv_graph_auto_collect_db.bool_ ()) {
+      return;
+   }
+
+   worker.enqueue ([this] () {
+      syncCollectOnline ();
+   });
+}
+
 void BotGraph::calculatePathRadius (int index) {
    // calculate "wayzones" for the nearest node  (meaning a dynamic distance area to vary node origin)
 
@@ -1658,6 +1782,9 @@ bool BotGraph::loadGraphData () {
       }
       cv_debug_goal.set (kInvalidNodeIndex);
 
+      // try to do graph collection, and push them to graph database automatically
+      collectOnline ();
+
       return true;
    }
    else {

src/storage.cpp

@@ -49,7 +49,7 @@ template <typename U> bool BotStorage::load (SmallArray <U> &data, ExtenHeader *
       auto downloadAddress = cv_graph_url.str ();
 
       auto toDownload = buildPath (storageToBotFile (type.option), false);
-      auto fromDownload = strings.format ("http://%s/graph/%s.graph", downloadAddress, lowercaseMapName);
+      auto fromDownload = strings.format ("%s://%s/graph/%s.graph", product.httpScheme, downloadAddress, lowercaseMapName);
 
       // try to download
       if (http.downloadFile (fromDownload, toDownload)) {
@@ -301,7 +301,7 @@ template <typename U> BotStorage::SaveLoadData BotStorage::guessType () {
 
 #else 
 
-String BotStorage::buildPath (int32_t file, bool isMemoryLoad) {
+String BotStorage::buildPath (int32_t file, bool isMemoryLoad, bool withoutMapName) {
    using FilePath = Twin <String, String>;
 
    static HashMap <int32_t, FilePath> paths = {
@@ -342,7 +342,7 @@ String BotStorage::buildPath (int32_t file, bool isMemoryLoad) {
       strftime (timebuf, StringBuffer::StaticBufferSize, "L%d%m%Y", &timeinfo);
       path.emplace (strings.format ("%s_%s.%s", product.nameLower, timebuf, paths[file].second));
    }
-   else {
+   else if (!withoutMapName) {
       String mapName = game.getMapName ();
       path.emplace (strings.format ("%s.%s", mapName.lowercase (), paths[file].second));
    }