From ee79b0750ccfa49b6211f9e0709a94f55839a53b Mon Sep 17 00:00:00 2001 From: Ceikry Date: Mon, 31 Jul 2023 14:59:53 +0000 Subject: [PATCH] Grafana Integration + Unified SQLite Boilerplate + Fixed bot movement performance regression --- Server/db_exports/grafana-dashboard.json | 539 ++++++++++++++++++ Server/src/main/core/ServerConstants.kt | 9 + .../main/core/game/bots/GeneralBotCreator.kt | 39 +- Server/src/main/core/game/ge/GEDB.kt | 197 ++----- .../core/game/interaction/MovementPulse.java | 6 +- .../node/entity/player/info/PlayerMonitor.kt | 106 ++-- .../game/system/config/ServerConfigParser.kt | 11 +- .../src/main/core/game/world/PulseRunner.kt | 20 + .../core/game/world/update/UpdateSequence.kt | 11 +- .../main/core/integrations/grafana/Grafana.kt | 158 +++++ .../integrations/sqlite/SQLiteProvider.kt | 94 +++ .../src/main/core/worker/MajorUpdateWorker.kt | 8 +- Server/worldprops/default.conf | 9 + 13 files changed, 984 insertions(+), 223 deletions(-) create mode 100644 Server/db_exports/grafana-dashboard.json create mode 100644 Server/src/main/core/integrations/grafana/Grafana.kt create mode 100644 Server/src/main/core/integrations/sqlite/SQLiteProvider.kt diff --git a/Server/db_exports/grafana-dashboard.json b/Server/db_exports/grafana-dashboard.json new file mode 100644 index 000000000..5d4b68c72 --- /dev/null +++ b/Server/db_exports/grafana-dashboard.json @@ -0,0 +1,539 @@ +{ + "__inputs": [ + { + "name": "DS_SQLITE", + "label": "SQLite", + "description": "", + "type": "datasource", + "pluginId": "frser-sqlite-datasource", + "pluginName": "SQLite" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "frser-sqlite-datasource", + "name": "SQLite", + "version": "3.3.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.1" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "misc_pulses" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packet_incoming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.1", + "targets": [ + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "queryText": "SELECT * FROM tick_lengths", + "queryType": "table", + "rawQueryText": "SELECT * FROM tick_lengths", + "refId": "A", + "timeColumns": [ + "ts" + ] + } + ], + "title": "Tick Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 13, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.1", + "targets": [ + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "queryText": "select * from top_pulses;", + "queryType": "table", + "rawQueryText": "select * from top_pulses;", + "refId": "A", + "timeColumns": [ + "time", + "ts" + ] + } + ], + "title": "Tick Time By Pulse (Total)", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [ + { + "path": "pulses" + } + ], + "keepTime": false, + "replace": false, + "source": "pulse_info" + } + }, + { + "id": "extractFields", + "options": { + "source": "pulses" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 13, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "queryText": "SELECT * FROM high_volume_pulses;", + "queryType": "table", + "rawQueryText": "SELECT * FROM high_volume_pulses;", + "refId": "A", + "timeColumns": [ + "time", + "ts" + ] + } + ], + "title": "High Volume Pulses", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "source": "pulse_info" + } + }, + { + "id": "extractFields", + "options": { + "source": "pulses" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 13, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "frser-sqlite-datasource", + "uid": "${DS_SQLITE}" + }, + "queryText": "SELECT * FROM bot_counts;", + "queryType": "table", + "rawQueryText": "SELECT * FROM bot_counts;", + "refId": "A", + "timeColumns": [ + "time", + "ts" + ] + } + ], + "title": "Bot Population", + "type": "timeseries" + } + ], + "refresh": "1h", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Main", + "uid": "ffccf845-00a2-46f2-b0c0-a8b88032c701", + "version": 10, + "weekStart": "" +} \ No newline at end of file diff --git a/Server/src/main/core/ServerConstants.kt b/Server/src/main/core/ServerConstants.kt index 3d8def976..016f7d352 100644 --- a/Server/src/main/core/ServerConstants.kt +++ b/Server/src/main/core/ServerConstants.kt @@ -91,6 +91,9 @@ class ServerConstants { @JvmField var ASDT_DATA_PATH: String? = null + @JvmField + var GRAFANA_PATH: String? = null + //the max number of players. @JvmField var MAX_PLAYERS = 2000 @@ -168,6 +171,12 @@ class ServerConstants { @JvmField var JAVA_VERSION = 8 + @JvmField + var GRAFANA_LOGGING = true + + @JvmField + var GRAFANA_TTL_DAYS = 7 + //location names for the ::to command. val TELEPORT_DESTINATIONS = arrayOf( arrayOf(Location.create(2974, 4383, 2), "corp", "corporal", "corporeal"), diff --git a/Server/src/main/core/game/bots/GeneralBotCreator.kt b/Server/src/main/core/game/bots/GeneralBotCreator.kt index 42378dd4c..361247a36 100644 --- a/Server/src/main/core/game/bots/GeneralBotCreator.kt +++ b/Server/src/main/core/game/bots/GeneralBotCreator.kt @@ -1,12 +1,21 @@ package core.game.bots -import content.global.bots.Idler -import core.Server import core.game.node.entity.player.Player import core.game.system.task.Pulse import core.game.world.GameWorld import core.game.world.map.Location import core.tools.RandomFunction +import core.Server +import content.global.bots.Idler +import core.api.* +import core.game.interaction.MovementPulse +import core.tools.Log + +class GBCTick : TickListener { + override fun tick() { + GeneralBotCreator.botPulsesTriggeredThisTick = 0 + } +} class GeneralBotCreator { //org/crandor/net/packet/in/InteractionPacket.java <<< This is a very useful class for learning to code bots @@ -28,18 +37,34 @@ class GeneralBotCreator { player.setAttribute("botting:script",pulse) } - inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1){ + companion object { + var botPulsesTriggeredThisTick = 0 + } + + inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1) { var ticks = 0 init { botScript.init(isPlayer) } var randomDelay = 0 - + var lastBotLocation: Location = botScript.bot.location.transform(0,0,0) + var lastBotMoveTicks = getWorldTicks() override fun pulse(): Boolean { if(randomDelay > 0){ randomDelay -= 1 return false } + if (botScript.bot.pulseManager.hasPulseRunning()) { + if (botScript.bot.pulseManager.current is MovementPulse) { + if (botScript.bot.location != lastBotLocation) { + lastBotLocation = botScript.bot.location.transform(0,0,0) + lastBotMoveTicks = getWorldTicks() + } + if (lastBotLocation == botScript.bot.location && getWorldTicks() - lastBotMoveTicks > 5) { + botScript.bot.pulseManager.current.stop() + } + } + } /* * Chatboxes and interfaces will cause the authentic interaction subsystem @@ -73,10 +98,12 @@ class GeneralBotCreator { }*/ if(!botScript.running) return true //has to be separated this way or it double-submits the respawn pulse. + if (botPulsesTriggeredThisTick++ >= 50) + return false - val idleRoll = RandomFunction.random(50) + val idleRoll = RandomFunction.random(10) if(idleRoll == 2 && botScript !is Idler){ - randomDelay += RandomFunction.random(2,20) + randomDelay += RandomFunction.random(20,50) return false } botScript.tick() diff --git a/Server/src/main/core/game/ge/GEDB.kt b/Server/src/main/core/game/ge/GEDB.kt index 6db3ca4dc..3fedd1e2d 100644 --- a/Server/src/main/core/game/ge/GEDB.kt +++ b/Server/src/main/core/game/ge/GEDB.kt @@ -1,71 +1,33 @@ package core.game.ge +import core.ServerConstants import core.cache.def.impl.ItemDefinition +import core.integrations.sqlite.SQLiteProvider import org.json.simple.JSONArray import org.json.simple.JSONObject -import org.json.simple.parser.JSONParser -import org.sqlite.SQLiteDataSource -import core.ServerConstants -import java.io.File -import java.io.FileReader import java.sql.Connection -import java.util.concurrent.TimeUnit -import java.util.concurrent.locks.ReentrantLock /** * Collection of methods for interacting with the grand exchange databases * @author Ceikry */ object GEDB { - private var pathString = "" - public var connection: Connection? = null - private var initialized = false - private var connectionRefs = 0 - private var obtainConnectionLock = ReentrantLock() - private var dbRunLock = ReentrantLock() + lateinit var db: SQLiteProvider fun init() { - init(File(ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db").absolutePath) + init (ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db") } - //This needs to be a separate method, so we can call it after the server config has been parsed - fun init(path: String) - { - if(initialized) return - pathString = path - //Check if the grandexchange.db file already exists. If not, create it and create the tables. - if(!File(path).exists()) - generateAndTransfer() + fun init (path: String) { + db = SQLiteProvider(path, expectedTables) + db.initTables { createdTable: String -> + if (createdTable == "price_index") + populateInitialPriceIndex() + } } @JvmStatic fun run(closure: (conn: Connection) -> Unit) { - dbRunLock.tryLock(10000L, TimeUnit.MILLISECONDS) - - connectionRefs++ - val con = connect() - closure.invoke(con) - connectionRefs-- - - if(connectionRefs == 0) { - con.close() - } - - dbRunLock.unlock() - } - - private fun connect(): Connection - { - obtainConnectionLock.tryLock(10000L, TimeUnit.MILLISECONDS) - - if (connection == null || connection!!.isClosed) - { - val ds = SQLiteDataSource() - ds.url = "jdbc:sqlite:$pathString" - connection = ds.connection - } - - obtainConnectionLock.unlock() - return connection!! + db.run(closure) } private fun convertJsonArray(arr: JSONArray): String @@ -85,107 +47,48 @@ object GEDB { return sb.toString() } - private fun generateAndTransfer() + var expectedTables = hashMapOf( + "player_offers" to "CREATE TABLE player_offers(" + + "uid INTEGER PRIMARY KEY ASC," + + "player_uid INTEGER," + + "item_id INTEGER," + + "amount_total INTEGER," + + "amount_complete INTEGER," + + "offered_value INTEGER," + + "time_stamp INTEGER," + + "offer_state INTEGER," + + "is_sale INTEGER," + + "withdraw_items STRING ," + + "slot_index INTEGER," + + "total_coin_xc INTEGER)", + "bot_offers" to "CREATE TABLE bot_offers(" + + "item_id INTEGER," + + "amount INTEGER)", + "price_index" to "CREATE TABLE price_index(" + + "item_id INTEGER," + + "value INTEGER," + + "total_value INTEGER," + + "unique_trades INTEGER," + + "last_update INTEGER)" + ) + + private fun populateInitialPriceIndex() { - val conn = connect() //for sqlite jdbc, attempting to connect to a non-existing .db file creates it. - val statement = conn.createStatement() - - //table for tracking player offers - replaces offer_dispatch.json - statement.execute( - "CREATE TABLE player_offers(" + - "uid INTEGER PRIMARY KEY ASC," + - "player_uid INTEGER," + - "item_id INTEGER," + - "amount_total INTEGER," + - "amount_complete INTEGER," + - "offered_value INTEGER," + - "time_stamp INTEGER," + - "offer_state INTEGER," + - "is_sale INTEGER," + - "withdraw_items STRING ," + - "slot_index INTEGER," + - "total_coin_xc INTEGER)" - ) - - //table for tracking bot offers - replaces bot_offers.json - statement.execute( - "CREATE TABLE bot_offers(" + - "item_id INTEGER," + - "amount INTEGER)" - ) - - //table for tracking price index data - replaces gedb.xml - statement.execute( - "CREATE TABLE price_index(" + - "item_id INTEGER," + - "value INTEGER," + - "total_value INTEGER," + - "unique_trades INTEGER," + - "last_update INTEGER)" - ) - - //check if the old .json and .xml files exist, and if they do, read them into the sqlite database and remove them. - val playerJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "offer_dispatch.json" - val botJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "bot_offers.json" - - if(File(playerJson).exists()) - { - val parser = JSONParser() - val reader = FileReader(playerJson) - val saveFile = parser.parse(reader) as JSONObject - - if (saveFile.containsKey("offers")) { - val offers = saveFile["offers"] as JSONArray - - for (offer in offers) { - val o = offer as JSONObject - statement.execute("insert into player_offers(player_uid,item_id,amount_total,amount_complete,offered_value,time_stamp,offer_state,is_sale,withdraw_items,total_coin_xc,slot_index) " + - "values(" + - "${o["playerUID"]}," + - "${o["itemId"]}," + - "${o["amount"]}," + - "${o["completedAmount"]}," + - "${o["offeredValue"]}," + - "${o["timeStamp"]}," + - "${o["offerState"]}," + - "${if (o["sale"] as Boolean) 1 else 0}," + - "'" + convertJsonArray(o["withdrawItems"] as JSONArray) + "'," + - "${o["totalCoinExchange"]}," + - "-1)" - ) - - } - } - - reader.close() - //File(playerJson).delete() - } - - if(File(botJson).exists()) - { - val parser = JSONParser() - val reader = FileReader(botJson) - val saveFile = parser.parse(reader) as JSONObject - - if (saveFile.containsKey("offers")) { - val offers = saveFile["offers"] as JSONArray - - for (offer in offers) { - val o = offer as JSONObject - statement.execute("insert into bot_offers " + - "values(${o["item"]},${o["qty"]})") - } - } - reader.close() - } - statement.close() - - //price index isn't worth transferring, so we're just going to make a new one. - ItemDefinition.getDefinitions().values.forEach { def -> - if(def.isTradeable){ - PriceIndex.allowItem(def.id) + run { + with (it.prepareStatement(PriceIndex.INSERT_QUERY)) { + ItemDefinition.getDefinitions().values.forEach { def -> + if(def.isTradeable) { + setInt(1, def.id) + setInt(2, def.getAlchemyValue(true)) + setInt(3, def.getAlchemyValue(true)) + setInt(4, 0) + setInt(5, 0) + execute() + } + } } } + } } \ No newline at end of file diff --git a/Server/src/main/core/game/interaction/MovementPulse.java b/Server/src/main/core/game/interaction/MovementPulse.java index 3bc506e5b..89f05344b 100644 --- a/Server/src/main/core/game/interaction/MovementPulse.java +++ b/Server/src/main/core/game/interaction/MovementPulse.java @@ -192,6 +192,8 @@ public abstract class MovementPulse extends Pulse { if (!mover.getViewport().getRegion().isActive()) return false; + if (!isRunning()) return true; + if (!validate()) { stop(); return true; @@ -295,6 +297,8 @@ public abstract class MovementPulse extends Pulse { if (destination instanceof Entity || interactLocation == null || (!mover.getWalkingQueue().hasPath() && interactLocation.getDistance(mover.getLocation()) > 0) || (usingTruncatedPath && destination.getLocation().getDistance(mover.getLocation()) < 14)) { if (!checkAllowMovement()) return; + if (destination instanceof Entity && previousLoc != null && previousLoc.equals(loc) && mover.getWalkingQueue().hasPath()) + return; Path path; Pair truncation = truncateLoc(mover, loc != null ? loc : destination.getLocation()); @@ -401,7 +405,7 @@ public abstract class MovementPulse extends Pulse { Vector pathDiff = Vector.betweenLocs (destination.getLocation(), Location.create(p.getX(), p.getY(), plane)); Location predictedCenterPos = (destination.getMathematicalCenter().plus(pathDiff)).toLocation(plane); Vector toPlayerNormalized = Vector.betweenLocs(predictedCenterPos, mover.getCenterLocation()).normalized(); - return predictedCenterPos.transform(toPlayerNormalized.times(destination.size())); + return predictedCenterPos.transform(toPlayerNormalized.times(destination.size() + 1)); } diff --git a/Server/src/main/core/game/node/entity/player/info/PlayerMonitor.kt b/Server/src/main/core/game/node/entity/player/info/PlayerMonitor.kt index 8e0a1d950..112d51a3c 100644 --- a/Server/src/main/core/game/node/entity/player/info/PlayerMonitor.kt +++ b/Server/src/main/core/game/node/entity/player/info/PlayerMonitor.kt @@ -1,16 +1,15 @@ package core.game.node.entity.player.info +import core.ServerConstants import core.api.getItemName import core.game.container.Container import core.game.node.entity.player.Player import core.game.node.entity.skill.Skills import core.integrations.discord.Discord -import kotlinx.coroutines.* -import org.sqlite.SQLiteDataSource -import core.ServerConstants +import core.integrations.sqlite.SQLiteProvider import core.tools.Log -import core.tools.SystemLogger -import java.io.File +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel import java.sql.Connection import java.util.concurrent.LinkedBlockingQueue import kotlin.math.abs @@ -18,6 +17,25 @@ import kotlin.math.abs object PlayerMonitor { private val eventQueue = LinkedBlockingQueue() private var activeTask: Job? = null + private lateinit var db: SQLiteProvider + + var expectedTables = hashMapOf( + "chat_logs" to """ + CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC ); + """, + "misc_logs" to """ + CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC); + """, + "trade_logs" to """ + CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC ); + """, + "xp_gains" to """ + CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC) + """, + "wealth_logs" to """ + CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC ) + """ + ) @JvmStatic fun logWealthChange(player: Player, totalWealth: Long, diff: Long) { val event = LogEvent.WealthLog( @@ -120,72 +138,34 @@ object PlayerMonitor { } } - fun processQueuedEvents() { - val path = ServerConstants.LOGS_PATH + "playerlogs.db" - if (activeTask?.isActive == true) - return - if (!File(path).exists()) { - createSqliteDatabase(path) - } - activeTask = GlobalScope.launch { - val conn = connect(path) - conn.use { - while (eventQueue.isNotEmpty()) { - val event = withContext(Dispatchers.IO) { eventQueue.take() } - process(event, it) - } - } + fun init () { + if (!this::db.isInitialized) { + var path = ServerConstants.LOGS_PATH + "playerlogs.db" + db = SQLiteProvider(path, expectedTables) + db.initTables() } } - @JvmStatic fun flushRemainingEventsImmediately() { - core.api.log(this::class.java, Log.FINE, "Flushing player log events...") - val path = ServerConstants.LOGS_PATH + "playerlogs.db" - if (!File(path).exists()) { - createSqliteDatabase(path) - } - if (activeTask != null) - activeTask?.cancel("Interrupted by shutdown. This is probably fine.") - val conn = connect(path) - conn.use { + fun processQueuedEvents() { + init() + if (activeTask?.isActive == true) + return + + activeTask = db.runAsync { while (eventQueue.isNotEmpty()) process(eventQueue.take(), it) } } - private fun connect(path: String) : Connection { - val source = SQLiteDataSource() - source.url = "jdbc:sqlite:$path" - return source.connection - } - - private fun createSqliteDatabase(path: String) { - if (!File(ServerConstants.LOGS_PATH).exists()) - File(ServerConstants.LOGS_PATH).mkdirs() - val connection = connect(path) - val stmt = connection.createStatement() - stmt.execute( - """ - CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC ); - """) - stmt.execute( - """ - CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC); - """) - stmt.execute( - """ - CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC ); - """) - stmt.execute( - """ - CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC) - """) - stmt.execute( - """ - CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC ) - """) - stmt.close() - connection.close() + @JvmStatic fun flushRemainingEventsImmediately() { + core.api.log(this::class.java, Log.FINE, "Flushing player log events...") + init() + if (activeTask != null) + activeTask?.cancel("Interrupted by shutdown. This is probably fine.") + db.run { + while (eventQueue.isNotEmpty()) + process(eventQueue.take(), it) + } } private fun process(event: LogEvent, conn: Connection) { diff --git a/Server/src/main/core/game/system/config/ServerConfigParser.kt b/Server/src/main/core/game/system/config/ServerConfigParser.kt index 0dd445649..f422e01b1 100644 --- a/Server/src/main/core/game/system/config/ServerConfigParser.kt +++ b/Server/src/main/core/game/system/config/ServerConfigParser.kt @@ -139,16 +139,16 @@ object ServerConfigParser { ServerConstants.BANK_BOOTH_QUICK_OPEN = data.getBoolean("world.bank_booth_quick_open", false) ServerConstants.BANK_BOOTH_NOTE_ENABLED = data.getBoolean("world.bank_booth_note_enabled", true) ServerConstants.BANK_BOOTH_NOTE_UIM = data.getBoolean("world.bank_booth_note_uim", true) - ServerConstants.DISCORD_GE_WEBHOOK = data.getString("server.discord_webhook", "") + ServerConstants.DISCORD_GE_WEBHOOK = data.getString("integrations.discord_ge_webhook", "") ServerConstants.WATCHDOG_ENABLED = data.getBoolean("server.watchdog_enabled", true) ServerConstants.I_AM_A_CHEATER = data.getBoolean("world.i_want_to_cheat", false) ServerConstants.USE_AUTH = data.getBoolean("server.use_auth", true) ServerConstants.PERSIST_ACCOUNTS = data.getBoolean("server.persist_accounts", true) ServerConstants.DAILY_ACCOUNT_LIMIT = data.getLong("server.daily_accounts_per_ip", 3L).toInt() - ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("server.moderation_webhook", "") + ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("integrations.discord_moderation_webhook", "") ServerConstants.NOAUTH_DEFAULT_ADMIN = data.getBoolean("server.noauth_default_admin", false) ServerConstants.DRAGON_AXE_USE_OSRS_SPEC = data.getBoolean("world.dragon_axe_use_osrs_spec", false) - ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("server.openrsc_integration_webhook", "") + ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("integrations.openrsc_integration_webhook", "") ServerConstants.ENABLE_GLOBALCHAT = data.getBoolean("world.enable_globalchat", true) ServerConstants.MAX_PATHFIND_DISTANCE = data.getLong("server.max_pathfind_dist", 25L).toInt() ServerConstants.IRONMAN_ICONS = data.getBoolean("world.ironman_icons", false) @@ -156,6 +156,9 @@ object ServerConfigParser { ServerConstants.PLAYER_STOCK_RECIRCULATE = data.getBoolean("world.playerstock_bot_offers", true) ServerConstants.BOTSTOCK_LIMIT = data.getLong("world.botstock_limit", 5000L).toInt() ServerConstants.BETTER_AGILITY_PYRAMID_GP = data.getBoolean("world.better_agility_pyramid_gp", true) + ServerConstants.GRAFANA_PATH = data.getPath("integrations.grafana_log_path") + ServerConstants.GRAFANA_LOGGING = data.getBoolean("integrations.grafana_logging", false) + ServerConstants.GRAFANA_TTL_DAYS = data.getLong("integrations.grafana_log_ttl_days", 7L).toInt() val logLevel = data.getString("server.log_level", "VERBOSE").uppercase() ServerConstants.LOG_LEVEL = parseEnumEntry(logLevel) ?: LogLevel.VERBOSE @@ -164,7 +167,7 @@ object ServerConfigParser { private fun Toml.getPath(key: String): String{ try { - return parsePath(getString(key).replace("@data", ServerConstants.DATA_PATH!!)) + return parsePath(getString(key, "@data/").replace("@data", ServerConstants.DATA_PATH!!)) } catch (e: Exception){ log(this::class.java, Log.ERR, "Error parsing key: $key") exitProcess(0) diff --git a/Server/src/main/core/game/world/PulseRunner.kt b/Server/src/main/core/game/world/PulseRunner.kt index 7ff62a3e3..e320cd7c9 100644 --- a/Server/src/main/core/game/world/PulseRunner.kt +++ b/Server/src/main/core/game/world/PulseRunner.kt @@ -5,7 +5,9 @@ import core.game.system.task.Pulse import core.game.bots.GeneralBotCreator import core.tools.Log import core.tools.SystemLogger +import core.ServerConstants import java.util.concurrent.LinkedBlockingQueue +import core.integrations.grafana.* class PulseRunner { private val pulses = LinkedBlockingQueue() @@ -19,6 +21,8 @@ class PulseRunner { fun updateAll() { val pulseCount = pulses.size + var totalTimeBotPulses = 0 + var totalTimeOtherPulses = 0 for (i in 0 until pulseCount) { val pulse = pulses.take() @@ -33,8 +37,24 @@ class PulseRunner { } } + var pulseName = pulse::class.java.name + + if (pulse is GeneralBotCreator.BotScriptPulse || pulseName.contains("ScriptAPI")) { + totalTimeBotPulses += elapsedTime.toInt() + } else { + totalTimeOtherPulses += elapsedTime.toInt() + } + + Grafana.addPulseLength (pulseName, elapsedTime.toInt()) + Grafana.countPulse (pulseName) + notifyIfTooLong(pulse, elapsedTime) } + + if (ServerConstants.GRAFANA_LOGGING) { + Grafana.botPulseTime = totalTimeBotPulses + Grafana.otherPulseTime = totalTimeOtherPulses + } } private fun measure(logic: () -> Unit): Long { diff --git a/Server/src/main/core/game/world/update/UpdateSequence.kt b/Server/src/main/core/game/world/update/UpdateSequence.kt index 0c0e19448..e50650f70 100644 --- a/Server/src/main/core/game/world/update/UpdateSequence.kt +++ b/Server/src/main/core/game/world/update/UpdateSequence.kt @@ -9,6 +9,7 @@ import core.net.packet.PacketRepository import core.net.packet.context.PlayerContext import core.net.packet.out.ClearMinimapFlag import core.game.world.repository.Repository +import core.integrations.grafana.* /** * The entity update sequence. @@ -32,15 +33,23 @@ class UpdateSequence playersList = renderablePlayers npcList = Repository.renderableNpcs lobbyList!!.map{ PacketRepository.send(ClearMinimapFlag::class.java, PlayerContext(it)) } + + var playerTickStart = System.currentTimeMillis() renderablePlayers.forEach(Player::tick) + Grafana.playerTickTime = (System.currentTimeMillis() - playerTickStart).toInt() + + var npcTickStart = System.currentTimeMillis() npcList!!.forEach(NPC::tick) + Grafana.npcTickTime = (System.currentTimeMillis() - npcTickStart).toInt() } /** * Runs the updating part of the sequence. */ fun run() { + var playerRenderStart = System.currentTimeMillis() renderablePlayers.forEach(Player::update) + Grafana.playerRenderTime = (System.currentTimeMillis() - playerRenderStart).toInt() } /** @@ -72,4 +81,4 @@ class UpdateSequence val renderablePlayers = InitializingNodeList() } -} \ No newline at end of file +} diff --git a/Server/src/main/core/integrations/grafana/Grafana.kt b/Server/src/main/core/integrations/grafana/Grafana.kt new file mode 100644 index 000000000..a78e75ec8 --- /dev/null +++ b/Server/src/main/core/integrations/grafana/Grafana.kt @@ -0,0 +1,158 @@ +package core.integrations.grafana + +import core.ServerConstants +import core.api.StartupListener +import core.game.bots.AIRepository +import core.integrations.sqlite.SQLiteProvider +import org.json.simple.JSONObject + +class Grafana : StartupListener { + override fun startup() { + if (!ServerConstants.GRAFANA_LOGGING) return + db = SQLiteProvider (ServerConstants.GRAFANA_PATH + "grafana.db", expectedTables) + db.initTables() + db.pruneOldData(ServerConstants.GRAFANA_TTL_DAYS) + } + + data class GrafanaData ( + val playerTickTime: Int, + val npcTickTime: Int, + val playerRenderTime: Int, + val totalTickTime: Int, + val packetProcessTime: Int, + val botPulseTime: Int, + val otherPulseTime: Int, + val pulseTimes: Array>, + val pulseCounts: Array> + ) + + companion object { + var playerTickTime = 0 + var npcTickTime = 0 + var playerRenderTime = 0 + var totalTickTime = 0 + var packetProcessTime = 0 + var botPulseTime = 0 + var otherPulseTime = 0 + lateinit var db: SQLiteProvider + + var pulseTimes = HashMap() + var pulseCounts = HashMap() + + fun addPulseLength(pulseName: String, time: Int) { + if (!this::db.isInitialized) return + pulseTimes[pulseName] = (pulseTimes[pulseName] ?: 0) + time + } + + fun countPulse(pulseName: String) { + if (!this::db.isInitialized) return + pulseCounts[pulseName] = (pulseCounts[pulseName] ?: 0) + 1 + } + + fun startTick() { + if (!this::db.isInitialized) return + playerTickTime = 0 + npcTickTime = 0 + playerRenderTime = 0 + totalTickTime = 0 + packetProcessTime = 0 + botPulseTime = 0 + otherPulseTime = 0 + pulseTimes.clear() + pulseCounts.clear() + } + + fun endTick() { + if (!this::db.isInitialized) return + val cycleData = GrafanaData ( + playerTickTime, + npcTickTime, + playerRenderTime, + totalTickTime, + packetProcessTime, + botPulseTime, + otherPulseTime, + pulseTimes.entries.toTypedArray(), + pulseCounts.entries.toTypedArray() + ) + db.runAsync { it -> + with (it.prepareStatement(INSERT_TOP_PULSES)) { + val topSorted = cycleData.pulseTimes.sortedByDescending { entry -> entry.value } + val rootObj = JSONObject() + val contentObj = JSONObject() + for (i in 0 until 5) { + contentObj [topSorted[i].key] = topSorted[i].value + } + rootObj["pulses"] = contentObj + setString(1, rootObj.toJSONString()) + setInt(2, getNowTime()) + execute() + } + + with (it.prepareStatement(INSERT_PULSE_COUNT)) { + val topSorted = cycleData.pulseCounts.sortedByDescending { entry -> entry.value } + val rootObj = JSONObject() + val contentObj = JSONObject() + for (i in 0 until 5) { + contentObj [topSorted[i].key] = topSorted[i].value + } + rootObj["pulses"] = contentObj + setString(1, rootObj.toJSONString()) + setInt(2, getNowTime()) + execute() + } + + with (it.prepareStatement(INSERT_BOT_COUNT)) { + setInt(1, AIRepository.PulseRepository.size) + setInt(2, getNowTime()) + execute() + } + + with (it.prepareStatement(INSERT_TICK_MEAS)) { + setInt(1, cycleData.botPulseTime) + setInt(2, cycleData.otherPulseTime) + setInt(3, cycleData.npcTickTime) + setInt(4, cycleData.playerTickTime) + setInt(5, cycleData.playerRenderTime) + setInt(6, cycleData.packetProcessTime) + setInt(7, cycleData.totalTickTime - (cycleData.botPulseTime + cycleData.otherPulseTime + cycleData.npcTickTime + cycleData.playerTickTime + cycleData.playerRenderTime + cycleData.packetProcessTime)) + setInt(8, getNowTime()) + execute() + } + } + } + + private fun getNowTime () : Int { + return (System.currentTimeMillis() / 1000L).toInt() + } + + private const val INSERT_TOP_PULSES = "INSERT INTO top_pulses (pulse_info, ts) VALUES (?, ?);" + private const val INSERT_PULSE_COUNT = "INSERT INTO high_volume_pulses (pulse_info, ts) VALUES (?, ?);" + private const val INSERT_BOT_COUNT = "INSERT INTO bot_counts (count, ts) VALUES (?, ?);" + private const val INSERT_TICK_MEAS = "INSERT INTO tick_lengths (bot_pulses, misc_pulses, npc_tick, player_tick, player_render, packet_incoming, other, ts) VALUES (?,?,?,?,?,?,?,?);" + private var expectedTables = hashMapOf( + "bot_counts" to "CREATE TABLE \"bot_counts\" (\n" + + "\t\"count\"\tINTEGER,\n" + + "\t\"ts\"\tINTEGER\n" + + ")", + "high_volume_pulses" to "CREATE TABLE \"high_volume_pulses\" (\n" + + "\t\"pulse_info\"\tTEXT,\n" + + "\t\"ts\"\tINTEGER\n" + + ")", + "tick_lengths" to "CREATE TABLE \"tick_lengths\" (\n" + + "\t\"bot_pulses\"\tINTEGER,\n" + + "\t\"misc_pulses\"\tINTEGER,\n" + + "\t\"npc_tick\"\tINTEGER,\n" + + "\t\"player_tick\"\tINTEGER,\n" + + "\t\"player_render\"\tINTEGER,\n" + + "\t\"packet_incoming\"\tINTEGER,\n" + + "\t\"other\"\tINTEGER,\n" + + "\t\"ts\"\tINTEGER\n" + + ")", + "top_pulses" to "CREATE TABLE \"top_pulses\" (\n" + + "\t\"pulse_info\"\tTEXT,\n" + + "\t\"ts\"\tINTEGER\n" + + ")" + ) + } +} \ No newline at end of file diff --git a/Server/src/main/core/integrations/sqlite/SQLiteProvider.kt b/Server/src/main/core/integrations/sqlite/SQLiteProvider.kt new file mode 100644 index 000000000..4155bf339 --- /dev/null +++ b/Server/src/main/core/integrations/sqlite/SQLiteProvider.kt @@ -0,0 +1,94 @@ +package core.integrations.sqlite + +import core.api.log +import core.tools.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch +import org.sqlite.SQLiteDataSource +import java.io.File +import java.sql.Connection +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantLock + +class SQLiteProvider (private val path: String, private val expectedTables: HashMap? = null) { + private var connection: Connection? = null + private var connectionRefs = 0 + private var obtainConnectionLock = ReentrantLock() + private var dbRunLock = ReentrantLock() + private val coroutineScope = CoroutineScope(Dispatchers.IO) + private lateinit var sqlitePath: String + + fun initTables(tableCreatedCallback: ((String) -> Unit)? = null) { + val pathTokens = File(path).absolutePath.split(File.separator) + val file = pathTokens.last() + val parentDir = pathTokens.dropLast(1).joinToString(File.separator) + + if (!File(parentDir).exists()) + File(parentDir).mkdirs() + sqlitePath = parentDir + File.separator + file + if (expectedTables?.isEmpty() != false) return + run { + with(it.prepareStatement(TABLE_CHECK)) { + for ((table, create) in expectedTables) { + setString(1, table) + val res = executeQuery() + if (!res.next()) + with(it.createStatement()) { execute(create); tableCreatedCallback?.invoke(table) } + } + } + } + } + + fun pruneOldData (daysToKeep: Int, timestampFieldName: String = "ts") { + if (expectedTables?.isEmpty() != false) return + val timeDiff = daysToKeep * 24 * 60 * 60 + val nowTime = System.currentTimeMillis() / 1000 + run { + with (it.createStatement()) { + for ((table, _) in expectedTables) { + execute("DELETE FROM $table WHERE $timestampFieldName <= ${nowTime - timeDiff};") + } + } + } + } + + fun runAsync (closure: (conn: Connection) -> Unit) : Job { + return coroutineScope.launch { run (closure) } + } + + fun run (closure: (conn: Connection) -> Unit) { + dbRunLock.tryLock(10000L, TimeUnit.MILLISECONDS) + + connectionRefs++ + val con = connect() + closure.invoke(con) + connectionRefs-- + + if(connectionRefs == 0) { + con.close() + } + + dbRunLock.unlock() + } + + private fun connect(): Connection + { + obtainConnectionLock.tryLock(10000L, TimeUnit.MILLISECONDS) + + if (connection == null || connection!!.isClosed) + { + val ds = SQLiteDataSource() + ds.url = "jdbc:sqlite:$sqlitePath" + connection = ds.connection + } + + obtainConnectionLock.unlock() + return connection!! + } + + companion object { + const val TABLE_CHECK = "SELECT name FROM sqlite_master WHERE type='table' AND name=?;" + } +} \ No newline at end of file diff --git a/Server/src/main/core/worker/MajorUpdateWorker.kt b/Server/src/main/core/worker/MajorUpdateWorker.kt index cee3aa942..e4553a6ec 100644 --- a/Server/src/main/core/worker/MajorUpdateWorker.kt +++ b/Server/src/main/core/worker/MajorUpdateWorker.kt @@ -19,6 +19,7 @@ import java.lang.Long.max import java.text.SimpleDateFormat import java.util.* import kotlin.system.exitProcess +import core.integrations.grafana.* /** * Handles the running of pulses and writing of masks, etc @@ -34,6 +35,7 @@ class MajorUpdateWorker { started = true Thread.sleep(600L) while (running) { + Grafana.startTick() val start = System.currentTimeMillis() Server.heartbeat() handleTickActions() @@ -77,6 +79,8 @@ class MajorUpdateWorker { } val end = System.currentTimeMillis() + Grafana.totalTickTime = (end - start).toInt() + Grafana.endTick() /* ServerMonitor.eventQueue.add(GuiEvent.UpdateTickTime(end - start)) ServerMonitor.eventQueue.add(GuiEvent.UpdatePulseCount(GameWorld.Pulser.TASKS.size))*/ Thread.sleep(max(600 - (end - start), 0)) @@ -87,7 +91,9 @@ class MajorUpdateWorker { fun handleTickActions(skipPulseUpdate: Boolean = false) { try { + var packetStart = System.currentTimeMillis() PacketProcessor.processQueue() + Grafana.packetProcessTime = (System.currentTimeMillis() - packetStart).toInt() //disconnect all players waiting to be disconnected Repository.disconnectionQueue.update() @@ -127,4 +133,4 @@ class MajorUpdateWorker { running = false worker.interrupt() } -} \ No newline at end of file +} diff --git a/Server/worldprops/default.conf b/Server/worldprops/default.conf index 546549699..b5f664ccd 100644 --- a/Server/worldprops/default.conf +++ b/Server/worldprops/default.conf @@ -31,6 +31,15 @@ database_password = "" database_address = "127.0.0.1" database_port = "3306" +[integrations] +grafana_logging = false +grafana_log_path = "@data/logs" +#how long grafana will keep old data for in days (anything older is pruned once on startup) +grafana_log_ttl_days = 7 +#discord_ge_webhook = "webhook link" +#discord_moderation_webhook = "webhook link" +#openrsc_integration_webhook = "webhook link" + [world] name = "2009scape"