From 46ebbeea57fb8e8fc00d60a0f1fc9ea508e3b682 Mon Sep 17 00:00:00 2001 From: jeefo Date: Tue, 30 Jan 2024 14:37:14 +0300 Subject: [PATCH] graph: allow graphs to be auto-collected (controlled via yb_graph_auto_collect_db) By default it's off, but it's allow bot to scan graph directory, do a diff with a graph db server and upload every single graph file that do not exist in central database. This is done in a separate thread and do not block server process, and only once server/game is started, not on change level. Also, it's not working on currently started map. --- cfg/addons/yapb/conf/yapb.cfg | 66 ++++++++++++++++-- ext/crlib | 2 +- inc/graph.h | 3 + inc/product.h | 1 + inc/storage.h | 2 +- src/control.cpp | 4 +- src/graph.cpp | 127 ++++++++++++++++++++++++++++++++++ src/storage.cpp | 6 +- 8 files changed, 200 insertions(+), 11 deletions(-) diff --git a/cfg/addons/yapb/conf/yapb.cfg b/cfg/addons/yapb/conf/yapb.cfg index 22c8959..944e30a 100644 --- a/cfg/addons/yapb/conf/yapb.cfg +++ b/cfg/addons/yapb/conf/yapb.cfg @@ -38,7 +38,7 @@ yb_graph_analyze_max_jump_height "44" // // The FPS at which analyzer process is running. This keeps game from freezing during analyzing. // --- -// Default: "30.0" +// Default: "30.0", Min: "25.0", Max: "99.0" // yb_graph_analyze_fps "30.0" @@ -345,6 +345,13 @@ yb_ignore_map_prefix_game_mode "0" // yb_threadpool_workers "-1" +// +// If enabled, bots will not apply throwing condition on grenades. +// --- +// Default: "0", Min: "0", Max: "1" +// +yb_grenadier_mode "0" + // // Specifies whether bot should not 'fix' camp directions of camp waypoints when loading old PWF format. // --- @@ -380,6 +387,13 @@ yb_graph_auto_save_count "15" // yb_graph_draw_distance "400" +// +// Allows bot's to exchange your graph files with graph database automatically. +// --- +// Default: "0", Min: "0", Max: "1" +// +yb_graph_auto_collect_db "0" + // // Kick bots to automatically make room for human players. // --- @@ -388,7 +402,7 @@ yb_graph_draw_distance "400" yb_autovacate "1" // -// How many slots autovacate feature should keep for human players +// How many slots autovacate feature should keep for human players. // --- // Default: "1", Min: "1", Max: "8" // @@ -541,6 +555,15 @@ yb_botskin_t "0" // yb_botskin_ct "0" +// +// Sets the default personality when creating bots with quota management. +// Allowed values: 'none', 'normal', 'careful', 'rusher'. +// If 'none' is specified personality chosen randomly. +// --- +// Default: "none" +// +yb_preferred_personality "none" + // // Lower bound for base bot ping shown in scoreboard upon creation. // --- @@ -597,12 +620,19 @@ yb_rotate_stay_min "360.0" // yb_rotate_stay_max "3600.0" +// +// When enabled, bots will not try to avoid teammates on their way. Assuming some of the semiclip plugins are in use. +// --- +// Default: "0", Min: "0", Max: "1" +// +yb_has_team_semiclip "0" + // // Selects the heuristic function mode. For debug purposes only. // --- -// Default: "3", Min: "0", Max: "4" +// Default: "0", Min: "0", Max: "4" // -yb_path_heuristic_mode "3" +yb_path_heuristic_mode "0" // // Limit maximum floyd-warshall memory (megabytes). Use Dijkstra if memory exceeds. @@ -625,6 +655,13 @@ yb_path_dijkstra_simple_distance "1" // yb_path_astar_post_smooth "0" +// +// Randomize pathfinding on each round start. +// --- +// Default: "1", Min: "0", Max: "1" +// +yb_path_randomize_on_round_start "1" + // // Enables or disables showing welcome message to host entity on game start. // --- @@ -646,6 +683,20 @@ yb_enable_query_hook "0" // yb_breakable_health_limit "500.0" +// +// Allows or disallows bots to return fake steam id. +// --- +// Default: "0", Min: "0", Max: "1" +// +yb_enable_fake_steamids "0" + +// +// Count player pings when calculating average ping for bots. If no, some random ping chosen for bots. +// --- +// Default: "1", Min: "0", Max: "1" +// +yb_count_players_for_fakeping "1" + // // Specifies whether bots able to use 'shift' if they thinks that enemy is near. // --- @@ -688,3 +739,10 @@ yb_random_knife_attacks "1" // yb_max_nodes_for_predict "25" +// +// Enables or disables extra hard difficulty for bots. +// --- +// Default: "0", Min: "0", Max: "1" +// +yb_whose_your_daddy "0" + diff --git a/ext/crlib b/ext/crlib index aa8e1e0..5225f85 160000 --- a/ext/crlib +++ b/ext/crlib @@ -1 +1 @@ -Subproject commit aa8e1e0eaed2e4b75e6b2a0fff5a68756cddc972 +Subproject commit 5225f857c44ced8e61e65b25892a0c8ab015250c diff --git a/inc/graph.h b/inc/graph.h index afdfb80..b529cb1 100644 --- a/inc/graph.h +++ b/inc/graph.h @@ -168,6 +168,7 @@ private: bool m_narrowChecked {}; bool m_silenceMessages {}; bool m_lightChecked {}; + bool m_isOnlineCollected {}; Vector m_learnVelocity {}; Vector m_learnPosition {}; @@ -259,6 +260,8 @@ public: void showStats (); void showFileInfo (); void emitNotify (int32_t sound); + void syncCollectOnline (); + void collectOnline (); IntArray getNearestInRadius (float radius, const Vector &origin, int maxCount = -1); const IntArray &getNodesInBucket (const Vector &pos); diff --git a/inc/product.h b/inc/product.h index c633299..9ed01c8 100644 --- a/inc/product.h +++ b/inc/product.h @@ -40,6 +40,7 @@ public: static constexpr StringRef url { "https://yapb.jeefo.net/" }; static constexpr StringRef download { "yapb.jeefo.net" }; static constexpr StringRef upload { "yapb.jeefo.net/upload" }; + static constexpr StringRef httpScheme { "http" }; static constexpr StringRef logtag { "YB" }; static constexpr StringRef dtime { __DATE__ " " __TIME__ }; static constexpr StringRef date { __DATE__ }; diff --git a/inc/storage.h b/inc/storage.h index 61237a5..20f7bcb 100644 --- a/inc/storage.h +++ b/inc/storage.h @@ -76,7 +76,7 @@ public: template bool error (bool isGraph, bool isDebug, MemFile &file, const char *fmt, Args &&...args); // builds the filename to requested filename - String buildPath (int32_t type, bool isMemoryLoad = false); + String buildPath (int32_t type, bool isMemoryLoad = false, bool withoutMapName = false); // get's relative path against bot library (bot library should reside in bin dir) StringRef getRunningPath (); diff --git a/src/control.cpp b/src/control.cpp index cf50df1..790dc8c 100644 --- a/src/control.cpp +++ b/src/control.cpp @@ -843,11 +843,11 @@ int BotControl::cmdNodeUpload () { String uploadUrlAddress = cv_graph_url_upload.str (); // only allow to upload to non-https endpoint - if (uploadUrlAddress.startsWith ("http")) { + if (uploadUrlAddress.startsWith ("https")) { msg ("Value of \"%s\" cvar should not contain URL scheme, only the host name and path.", cv_graph_url_upload.name ()); return BotCommandResult::Handled; } - String uploadUrl = strings.format ("http://%s", uploadUrlAddress); + String uploadUrl = strings.format ("%s://%s", product.httpScheme, uploadUrlAddress); msg ("\n"); msg ("WARNING!"); diff --git a/src/graph.cpp b/src/graph.cpp index 43ee410..8131a4c 100644 --- a/src/graph.cpp +++ b/src/graph.cpp @@ -12,6 +12,7 @@ ConVar cv_graph_url ("graph_url", product.download.chars (), "Specifies the URL ConVar cv_graph_url_upload ("graph_url_upload", product.upload.chars (), "Specifies the URL to which bots will try to upload the graph file to database.", false, 0.0f, 0.0f); ConVar cv_graph_auto_save_count ("graph_auto_save_count", "15", "Every N graph nodes placed on map, the graph will be saved automatically (without checks).", true, 0.0f, kMaxNodes); ConVar cv_graph_draw_distance ("graph_draw_distance", "400", "Maximum distance to draw graph nodes from editor viewport.", true, 64.0f, 3072.0f); +ConVar cv_graph_auto_collect_db ("graph_auto_collect_db", "0", "Allows bot's to exchange your graph files with graph database automatically."); void BotGraph::reset () { // this function initialize the graph structures.. @@ -1277,6 +1278,129 @@ void BotGraph::emitNotify (int32_t sound) { } } +void BotGraph::syncCollectOnline () { + m_isOnlineCollected = true; // once per server start + + // path to graph files + auto graphFilesPath = bstor.buildPath (BotFile::Graph, false, true); + + // enumerate graph files + FileEnumerator enumerator { strings.joinPath (graphFilesPath, "*.graph") }; + + // listing of graphs locally available + StringArray localGraphs {}; + + // collect all the files + while (enumerator) { + auto match = enumerator.getMatch (); + + match = match.substr (match.findLastOf (kPathSeparator) + 1); + match = match.substr (0, match.findFirstOf (".")); + + localGraphs.emplace (match); + enumerator.next (); + } + + // no graphs ? unbelievable + if (localGraphs.empty ()) { + return; + } + String uploadUrlAddress = cv_graph_url_upload.str (); + + // only allow to upload to non-https endpoint + if (uploadUrlAddress.startsWith ("https")) { + return; + } + String localFile = plat.tmpfname (); + + // don't forget remove temporary file + auto unlinkTemporary = [&] () { + if (plat.fileExists (localFile.chars ())) { + plat.removeFile (localFile.chars ()); + } + }; + + // write out our list of files into temporary + if (File lc { localFile, "wt" }) { + auto graphList = String::join (localGraphs, ","); + auto collectUrl = strings.format ("%s://%s/collect/%u", product.httpScheme, uploadUrlAddress, graphList.hash ()); + + lc.puts (graphList.chars ()); + lc.close (); + + // upload to collection diff + if (!http.uploadFile (collectUrl, localFile)) { + unlinkTemporary (); + return; + } + unlinkTemporary (); + + // download collection diff + if (!http.downloadFile (collectUrl, localFile)) { + return; + } + StringArray wanted {}; + + // decode answer + if (lc.open (localFile, "rt")) { + String lines; + + if (lc.getLine (lines)) { + wanted = lines.split (","); + } + lc.close (); + } + unlinkTemporary (); + + // if 'we're have something in diff, bailout + if (wanted.empty ()) { + return; + } + localGraphs.clear (); + + // convert graphs names into full paths + for (const auto &wn : wanted) { + if (wn == game.getMapName ()) { + continue; // skip current map always + } + localGraphs.emplace (strings.joinPath (graphFilesPath, wn) + ".graph"); + } + + // try to upload everything database wants + for (const auto &lg : localGraphs) { + if (!plat.fileExists (lg.chars ())) { + continue; + } + StorageHeader hdr {}; + + // read storage header and check if file NOT analyzed + if (File gp { lg, "rb" }) { + gp.read (&hdr, sizeof (StorageHeader)); + + // check the magic, graph is NOT analyzed and have some viable nodes number + if (hdr.magic == kStorageMagic && !(hdr.options & StorageOption::Analyzed) && hdr.length > 48) { + String uploadUrl = strings.format ("%s://%s", product.httpScheme, uploadUrlAddress); + + // try to upload to database (no need check if it's succeed) + http.uploadFile (uploadUrl, lg); + } + gp.close (); + } + } + } + unlinkTemporary (); +} + +void BotGraph::collectOnline () { + if (m_isOnlineCollected || !cv_graph_auto_collect_db.bool_ ()) { + return; + } + + worker.enqueue ([this] () { + syncCollectOnline (); + }); +} + void BotGraph::calculatePathRadius (int index) { // calculate "wayzones" for the nearest node (meaning a dynamic distance area to vary node origin) @@ -1658,6 +1782,9 @@ bool BotGraph::loadGraphData () { } cv_debug_goal.set (kInvalidNodeIndex); + // try to do graph collection, and push them to graph database automatically + collectOnline (); + return true; } else { diff --git a/src/storage.cpp b/src/storage.cpp index 3b17d8a..c41e73b 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -49,7 +49,7 @@ template bool BotStorage::load (SmallArray &data, ExtenHeader * auto downloadAddress = cv_graph_url.str (); auto toDownload = buildPath (storageToBotFile (type.option), false); - auto fromDownload = strings.format ("http://%s/graph/%s.graph", downloadAddress, lowercaseMapName); + auto fromDownload = strings.format ("%s://%s/graph/%s.graph", product.httpScheme, downloadAddress, lowercaseMapName); // try to download if (http.downloadFile (fromDownload, toDownload)) { @@ -301,7 +301,7 @@ template BotStorage::SaveLoadData BotStorage::guessType () { #else -String BotStorage::buildPath (int32_t file, bool isMemoryLoad) { +String BotStorage::buildPath (int32_t file, bool isMemoryLoad, bool withoutMapName) { using FilePath = Twin ; static HashMap paths = { @@ -342,7 +342,7 @@ String BotStorage::buildPath (int32_t file, bool isMemoryLoad) { strftime (timebuf, StringBuffer::StaticBufferSize, "L%d%m%Y", &timeinfo); path.emplace (strings.format ("%s_%s.%s", product.nameLower, timebuf, paths[file].second)); } - else { + else if (!withoutMapName) { String mapName = game.getMapName (); path.emplace (strings.format ("%s.%s", mapName.lowercase (), paths[file].second)); }