Grafana Integration + Unified SQLite Boilerplate + Fixed bot movement performance regression

This commit is contained in:
Ceikry
2023-07-31 14:59:53 +00:00
parent ac0da92a58
commit ee79b0750c
13 changed files with 984 additions and 223 deletions
+539
View File
@@ -0,0 +1,539 @@
{
"__inputs": [
{
"name": "DS_SQLITE",
"label": "SQLite",
"description": "",
"type": "datasource",
"pluginId": "frser-sqlite-datasource",
"pluginName": "SQLite"
}
],
"__elements": {},
"__requires": [
{
"type": "datasource",
"id": "frser-sqlite-datasource",
"name": "SQLite",
"version": "3.3.0"
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.0.1"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "misc_pulses"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "purple",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "other"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "packet_incoming"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "super-light-red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"queryText": "SELECT * FROM tick_lengths",
"queryType": "table",
"rawQueryText": "SELECT * FROM tick_lengths",
"refId": "A",
"timeColumns": [
"ts"
]
}
],
"title": "Tick Time",
"type": "timeseries"
},
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 13,
"x": 0,
"y": 9
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"queryText": "select * from top_pulses;",
"queryType": "table",
"rawQueryText": "select * from top_pulses;",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "Tick Time By Pulse (Total)",
"transformations": [
{
"id": "extractFields",
"options": {
"format": "json",
"jsonPaths": [
{
"path": "pulses"
}
],
"keepTime": false,
"replace": false,
"source": "pulse_info"
}
},
{
"id": "extractFields",
"options": {
"source": "pulses"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 13,
"y": 9
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"queryText": "SELECT * FROM high_volume_pulses;",
"queryType": "table",
"rawQueryText": "SELECT * FROM high_volume_pulses;",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "High Volume Pulses",
"transformations": [
{
"id": "extractFields",
"options": {
"format": "json",
"source": "pulse_info"
}
},
{
"id": "extractFields",
"options": {
"source": "pulses"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "stepBefore",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 11,
"x": 13,
"y": 18
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "frser-sqlite-datasource",
"uid": "${DS_SQLITE}"
},
"queryText": "SELECT * FROM bot_counts;",
"queryType": "table",
"rawQueryText": "SELECT * FROM bot_counts;",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "Bot Population",
"type": "timeseries"
}
],
"refresh": "1h",
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Main",
"uid": "ffccf845-00a2-46f2-b0c0-a8b88032c701",
"version": 10,
"weekStart": ""
}
+9
View File
@@ -91,6 +91,9 @@ class ServerConstants {
@JvmField
var ASDT_DATA_PATH: String? = null
@JvmField
var GRAFANA_PATH: String? = null
//the max number of players.
@JvmField
var MAX_PLAYERS = 2000
@@ -168,6 +171,12 @@ class ServerConstants {
@JvmField
var JAVA_VERSION = 8
@JvmField
var GRAFANA_LOGGING = true
@JvmField
var GRAFANA_TTL_DAYS = 7
//location names for the ::to command.
val TELEPORT_DESTINATIONS = arrayOf(
arrayOf(Location.create(2974, 4383, 2), "corp", "corporal", "corporeal"),
@@ -1,12 +1,21 @@
package core.game.bots
import content.global.bots.Idler
import core.Server
import core.game.node.entity.player.Player
import core.game.system.task.Pulse
import core.game.world.GameWorld
import core.game.world.map.Location
import core.tools.RandomFunction
import core.Server
import content.global.bots.Idler
import core.api.*
import core.game.interaction.MovementPulse
import core.tools.Log
class GBCTick : TickListener {
override fun tick() {
GeneralBotCreator.botPulsesTriggeredThisTick = 0
}
}
class GeneralBotCreator {
//org/crandor/net/packet/in/InteractionPacket.java <<< This is a very useful class for learning to code bots
@@ -28,18 +37,34 @@ class GeneralBotCreator {
player.setAttribute("botting:script",pulse)
}
inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1){
companion object {
var botPulsesTriggeredThisTick = 0
}
inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1) {
var ticks = 0
init {
botScript.init(isPlayer)
}
var randomDelay = 0
var lastBotLocation: Location = botScript.bot.location.transform(0,0,0)
var lastBotMoveTicks = getWorldTicks()
override fun pulse(): Boolean {
if(randomDelay > 0){
randomDelay -= 1
return false
}
if (botScript.bot.pulseManager.hasPulseRunning()) {
if (botScript.bot.pulseManager.current is MovementPulse) {
if (botScript.bot.location != lastBotLocation) {
lastBotLocation = botScript.bot.location.transform(0,0,0)
lastBotMoveTicks = getWorldTicks()
}
if (lastBotLocation == botScript.bot.location && getWorldTicks() - lastBotMoveTicks > 5) {
botScript.bot.pulseManager.current.stop()
}
}
}
/*
* Chatboxes and interfaces will cause the authentic interaction subsystem
@@ -73,10 +98,12 @@ class GeneralBotCreator {
}*/
if(!botScript.running) return true //has to be separated this way or it double-submits the respawn pulse.
if (botPulsesTriggeredThisTick++ >= 50)
return false
val idleRoll = RandomFunction.random(50)
val idleRoll = RandomFunction.random(10)
if(idleRoll == 2 && botScript !is Idler){
randomDelay += RandomFunction.random(2,20)
randomDelay += RandomFunction.random(20,50)
return false
}
botScript.tick()
+50 -147
View File
@@ -1,71 +1,33 @@
package core.game.ge
import core.ServerConstants
import core.cache.def.impl.ItemDefinition
import core.integrations.sqlite.SQLiteProvider
import org.json.simple.JSONArray
import org.json.simple.JSONObject
import org.json.simple.parser.JSONParser
import org.sqlite.SQLiteDataSource
import core.ServerConstants
import java.io.File
import java.io.FileReader
import java.sql.Connection
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
/**
* Collection of methods for interacting with the grand exchange databases
* @author Ceikry
*/
object GEDB {
private var pathString = ""
public var connection: Connection? = null
private var initialized = false
private var connectionRefs = 0
private var obtainConnectionLock = ReentrantLock()
private var dbRunLock = ReentrantLock()
lateinit var db: SQLiteProvider
fun init() {
init(File(ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db").absolutePath)
init (ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db")
}
//This needs to be a separate method, so we can call it after the server config has been parsed
fun init(path: String)
{
if(initialized) return
pathString = path
//Check if the grandexchange.db file already exists. If not, create it and create the tables.
if(!File(path).exists())
generateAndTransfer()
fun init (path: String) {
db = SQLiteProvider(path, expectedTables)
db.initTables { createdTable: String ->
if (createdTable == "price_index")
populateInitialPriceIndex()
}
}
@JvmStatic fun run(closure: (conn: Connection) -> Unit) {
dbRunLock.tryLock(10000L, TimeUnit.MILLISECONDS)
connectionRefs++
val con = connect()
closure.invoke(con)
connectionRefs--
if(connectionRefs == 0) {
con.close()
}
dbRunLock.unlock()
}
private fun connect(): Connection
{
obtainConnectionLock.tryLock(10000L, TimeUnit.MILLISECONDS)
if (connection == null || connection!!.isClosed)
{
val ds = SQLiteDataSource()
ds.url = "jdbc:sqlite:$pathString"
connection = ds.connection
}
obtainConnectionLock.unlock()
return connection!!
db.run(closure)
}
private fun convertJsonArray(arr: JSONArray): String
@@ -85,107 +47,48 @@ object GEDB {
return sb.toString()
}
private fun generateAndTransfer()
var expectedTables = hashMapOf(
"player_offers" to "CREATE TABLE player_offers(" +
"uid INTEGER PRIMARY KEY ASC," +
"player_uid INTEGER," +
"item_id INTEGER," +
"amount_total INTEGER," +
"amount_complete INTEGER," +
"offered_value INTEGER," +
"time_stamp INTEGER," +
"offer_state INTEGER," +
"is_sale INTEGER," +
"withdraw_items STRING ," +
"slot_index INTEGER," +
"total_coin_xc INTEGER)",
"bot_offers" to "CREATE TABLE bot_offers(" +
"item_id INTEGER," +
"amount INTEGER)",
"price_index" to "CREATE TABLE price_index(" +
"item_id INTEGER," +
"value INTEGER," +
"total_value INTEGER," +
"unique_trades INTEGER," +
"last_update INTEGER)"
)
private fun populateInitialPriceIndex()
{
val conn = connect() //for sqlite jdbc, attempting to connect to a non-existing .db file creates it.
val statement = conn.createStatement()
//table for tracking player offers - replaces offer_dispatch.json
statement.execute(
"CREATE TABLE player_offers(" +
"uid INTEGER PRIMARY KEY ASC," +
"player_uid INTEGER," +
"item_id INTEGER," +
"amount_total INTEGER," +
"amount_complete INTEGER," +
"offered_value INTEGER," +
"time_stamp INTEGER," +
"offer_state INTEGER," +
"is_sale INTEGER," +
"withdraw_items STRING ," +
"slot_index INTEGER," +
"total_coin_xc INTEGER)"
)
//table for tracking bot offers - replaces bot_offers.json
statement.execute(
"CREATE TABLE bot_offers(" +
"item_id INTEGER," +
"amount INTEGER)"
)
//table for tracking price index data - replaces gedb.xml
statement.execute(
"CREATE TABLE price_index(" +
"item_id INTEGER," +
"value INTEGER," +
"total_value INTEGER," +
"unique_trades INTEGER," +
"last_update INTEGER)"
)
//check if the old .json and .xml files exist, and if they do, read them into the sqlite database and remove them.
val playerJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "offer_dispatch.json"
val botJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "bot_offers.json"
if(File(playerJson).exists())
{
val parser = JSONParser()
val reader = FileReader(playerJson)
val saveFile = parser.parse(reader) as JSONObject
if (saveFile.containsKey("offers")) {
val offers = saveFile["offers"] as JSONArray
for (offer in offers) {
val o = offer as JSONObject
statement.execute("insert into player_offers(player_uid,item_id,amount_total,amount_complete,offered_value,time_stamp,offer_state,is_sale,withdraw_items,total_coin_xc,slot_index) " +
"values(" +
"${o["playerUID"]}," +
"${o["itemId"]}," +
"${o["amount"]}," +
"${o["completedAmount"]}," +
"${o["offeredValue"]}," +
"${o["timeStamp"]}," +
"${o["offerState"]}," +
"${if (o["sale"] as Boolean) 1 else 0}," +
"'" + convertJsonArray(o["withdrawItems"] as JSONArray) + "'," +
"${o["totalCoinExchange"]}," +
"-1)"
)
}
}
reader.close()
//File(playerJson).delete()
}
if(File(botJson).exists())
{
val parser = JSONParser()
val reader = FileReader(botJson)
val saveFile = parser.parse(reader) as JSONObject
if (saveFile.containsKey("offers")) {
val offers = saveFile["offers"] as JSONArray
for (offer in offers) {
val o = offer as JSONObject
statement.execute("insert into bot_offers " +
"values(${o["item"]},${o["qty"]})")
}
}
reader.close()
}
statement.close()
//price index isn't worth transferring, so we're just going to make a new one.
ItemDefinition.getDefinitions().values.forEach { def ->
if(def.isTradeable){
PriceIndex.allowItem(def.id)
run {
with (it.prepareStatement(PriceIndex.INSERT_QUERY)) {
ItemDefinition.getDefinitions().values.forEach { def ->
if(def.isTradeable) {
setInt(1, def.id)
setInt(2, def.getAlchemyValue(true))
setInt(3, def.getAlchemyValue(true))
setInt(4, 0)
setInt(5, 0)
execute()
}
}
}
}
}
}
@@ -192,6 +192,8 @@ public abstract class MovementPulse extends Pulse {
if (!mover.getViewport().getRegion().isActive())
return false;
if (!isRunning()) return true;
if (!validate()) {
stop();
return true;
@@ -295,6 +297,8 @@ public abstract class MovementPulse extends Pulse {
if (destination instanceof Entity || interactLocation == null || (!mover.getWalkingQueue().hasPath() && interactLocation.getDistance(mover.getLocation()) > 0) || (usingTruncatedPath && destination.getLocation().getDistance(mover.getLocation()) < 14)) {
if (!checkAllowMovement())
return;
if (destination instanceof Entity && previousLoc != null && previousLoc.equals(loc) && mover.getWalkingQueue().hasPath())
return;
Path path;
Pair<Boolean, Location> truncation = truncateLoc(mover, loc != null ? loc : destination.getLocation());
@@ -401,7 +405,7 @@ public abstract class MovementPulse extends Pulse {
Vector pathDiff = Vector.betweenLocs (destination.getLocation(), Location.create(p.getX(), p.getY(), plane));
Location predictedCenterPos = (destination.getMathematicalCenter().plus(pathDiff)).toLocation(plane);
Vector toPlayerNormalized = Vector.betweenLocs(predictedCenterPos, mover.getCenterLocation()).normalized();
return predictedCenterPos.transform(toPlayerNormalized.times(destination.size()));
return predictedCenterPos.transform(toPlayerNormalized.times(destination.size() + 1));
}
@@ -1,16 +1,15 @@
package core.game.node.entity.player.info
import core.ServerConstants
import core.api.getItemName
import core.game.container.Container
import core.game.node.entity.player.Player
import core.game.node.entity.skill.Skills
import core.integrations.discord.Discord
import kotlinx.coroutines.*
import org.sqlite.SQLiteDataSource
import core.ServerConstants
import core.integrations.sqlite.SQLiteProvider
import core.tools.Log
import core.tools.SystemLogger
import java.io.File
import kotlinx.coroutines.Job
import kotlinx.coroutines.cancel
import java.sql.Connection
import java.util.concurrent.LinkedBlockingQueue
import kotlin.math.abs
@@ -18,6 +17,25 @@ import kotlin.math.abs
object PlayerMonitor {
private val eventQueue = LinkedBlockingQueue<LogEvent>()
private var activeTask: Job? = null
private lateinit var db: SQLiteProvider
var expectedTables = hashMapOf(
"chat_logs" to """
CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC );
""",
"misc_logs" to """
CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC);
""",
"trade_logs" to """
CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC );
""",
"xp_gains" to """
CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC)
""",
"wealth_logs" to """
CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC )
"""
)
@JvmStatic fun logWealthChange(player: Player, totalWealth: Long, diff: Long) {
val event = LogEvent.WealthLog(
@@ -120,72 +138,34 @@ object PlayerMonitor {
}
}
fun processQueuedEvents() {
val path = ServerConstants.LOGS_PATH + "playerlogs.db"
if (activeTask?.isActive == true)
return
if (!File(path).exists()) {
createSqliteDatabase(path)
}
activeTask = GlobalScope.launch {
val conn = connect(path)
conn.use {
while (eventQueue.isNotEmpty()) {
val event = withContext(Dispatchers.IO) { eventQueue.take() }
process(event, it)
}
}
fun init () {
if (!this::db.isInitialized) {
var path = ServerConstants.LOGS_PATH + "playerlogs.db"
db = SQLiteProvider(path, expectedTables)
db.initTables()
}
}
@JvmStatic fun flushRemainingEventsImmediately() {
core.api.log(this::class.java, Log.FINE, "Flushing player log events...")
val path = ServerConstants.LOGS_PATH + "playerlogs.db"
if (!File(path).exists()) {
createSqliteDatabase(path)
}
if (activeTask != null)
activeTask?.cancel("Interrupted by shutdown. This is probably fine.")
val conn = connect(path)
conn.use {
fun processQueuedEvents() {
init()
if (activeTask?.isActive == true)
return
activeTask = db.runAsync {
while (eventQueue.isNotEmpty())
process(eventQueue.take(), it)
}
}
private fun connect(path: String) : Connection {
val source = SQLiteDataSource()
source.url = "jdbc:sqlite:$path"
return source.connection
}
private fun createSqliteDatabase(path: String) {
if (!File(ServerConstants.LOGS_PATH).exists())
File(ServerConstants.LOGS_PATH).mkdirs()
val connection = connect(path)
val stmt = connection.createStatement()
stmt.execute(
"""
CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC );
""")
stmt.execute(
"""
CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC);
""")
stmt.execute(
"""
CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC );
""")
stmt.execute(
"""
CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC)
""")
stmt.execute(
"""
CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC )
""")
stmt.close()
connection.close()
@JvmStatic fun flushRemainingEventsImmediately() {
core.api.log(this::class.java, Log.FINE, "Flushing player log events...")
init()
if (activeTask != null)
activeTask?.cancel("Interrupted by shutdown. This is probably fine.")
db.run {
while (eventQueue.isNotEmpty())
process(eventQueue.take(), it)
}
}
private fun process(event: LogEvent, conn: Connection) {
@@ -139,16 +139,16 @@ object ServerConfigParser {
ServerConstants.BANK_BOOTH_QUICK_OPEN = data.getBoolean("world.bank_booth_quick_open", false)
ServerConstants.BANK_BOOTH_NOTE_ENABLED = data.getBoolean("world.bank_booth_note_enabled", true)
ServerConstants.BANK_BOOTH_NOTE_UIM = data.getBoolean("world.bank_booth_note_uim", true)
ServerConstants.DISCORD_GE_WEBHOOK = data.getString("server.discord_webhook", "")
ServerConstants.DISCORD_GE_WEBHOOK = data.getString("integrations.discord_ge_webhook", "")
ServerConstants.WATCHDOG_ENABLED = data.getBoolean("server.watchdog_enabled", true)
ServerConstants.I_AM_A_CHEATER = data.getBoolean("world.i_want_to_cheat", false)
ServerConstants.USE_AUTH = data.getBoolean("server.use_auth", true)
ServerConstants.PERSIST_ACCOUNTS = data.getBoolean("server.persist_accounts", true)
ServerConstants.DAILY_ACCOUNT_LIMIT = data.getLong("server.daily_accounts_per_ip", 3L).toInt()
ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("server.moderation_webhook", "")
ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("integrations.discord_moderation_webhook", "")
ServerConstants.NOAUTH_DEFAULT_ADMIN = data.getBoolean("server.noauth_default_admin", false)
ServerConstants.DRAGON_AXE_USE_OSRS_SPEC = data.getBoolean("world.dragon_axe_use_osrs_spec", false)
ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("server.openrsc_integration_webhook", "")
ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("integrations.openrsc_integration_webhook", "")
ServerConstants.ENABLE_GLOBALCHAT = data.getBoolean("world.enable_globalchat", true)
ServerConstants.MAX_PATHFIND_DISTANCE = data.getLong("server.max_pathfind_dist", 25L).toInt()
ServerConstants.IRONMAN_ICONS = data.getBoolean("world.ironman_icons", false)
@@ -156,6 +156,9 @@ object ServerConfigParser {
ServerConstants.PLAYER_STOCK_RECIRCULATE = data.getBoolean("world.playerstock_bot_offers", true)
ServerConstants.BOTSTOCK_LIMIT = data.getLong("world.botstock_limit", 5000L).toInt()
ServerConstants.BETTER_AGILITY_PYRAMID_GP = data.getBoolean("world.better_agility_pyramid_gp", true)
ServerConstants.GRAFANA_PATH = data.getPath("integrations.grafana_log_path")
ServerConstants.GRAFANA_LOGGING = data.getBoolean("integrations.grafana_logging", false)
ServerConstants.GRAFANA_TTL_DAYS = data.getLong("integrations.grafana_log_ttl_days", 7L).toInt()
val logLevel = data.getString("server.log_level", "VERBOSE").uppercase()
ServerConstants.LOG_LEVEL = parseEnumEntry<LogLevel>(logLevel) ?: LogLevel.VERBOSE
@@ -164,7 +167,7 @@ object ServerConfigParser {
private fun Toml.getPath(key: String): String{
try {
return parsePath(getString(key).replace("@data", ServerConstants.DATA_PATH!!))
return parsePath(getString(key, "@data/").replace("@data", ServerConstants.DATA_PATH!!))
} catch (e: Exception){
log(this::class.java, Log.ERR, "Error parsing key: $key")
exitProcess(0)
@@ -5,7 +5,9 @@ import core.game.system.task.Pulse
import core.game.bots.GeneralBotCreator
import core.tools.Log
import core.tools.SystemLogger
import core.ServerConstants
import java.util.concurrent.LinkedBlockingQueue
import core.integrations.grafana.*
class PulseRunner {
private val pulses = LinkedBlockingQueue<Pulse>()
@@ -19,6 +21,8 @@ class PulseRunner {
fun updateAll() {
val pulseCount = pulses.size
var totalTimeBotPulses = 0
var totalTimeOtherPulses = 0
for (i in 0 until pulseCount) {
val pulse = pulses.take()
@@ -33,8 +37,24 @@ class PulseRunner {
}
}
var pulseName = pulse::class.java.name
if (pulse is GeneralBotCreator.BotScriptPulse || pulseName.contains("ScriptAPI")) {
totalTimeBotPulses += elapsedTime.toInt()
} else {
totalTimeOtherPulses += elapsedTime.toInt()
}
Grafana.addPulseLength (pulseName, elapsedTime.toInt())
Grafana.countPulse (pulseName)
notifyIfTooLong(pulse, elapsedTime)
}
if (ServerConstants.GRAFANA_LOGGING) {
Grafana.botPulseTime = totalTimeBotPulses
Grafana.otherPulseTime = totalTimeOtherPulses
}
}
private fun measure(logic: () -> Unit): Long {
@@ -9,6 +9,7 @@ import core.net.packet.PacketRepository
import core.net.packet.context.PlayerContext
import core.net.packet.out.ClearMinimapFlag
import core.game.world.repository.Repository
import core.integrations.grafana.*
/**
* The entity update sequence.
@@ -32,15 +33,23 @@ class UpdateSequence
playersList = renderablePlayers
npcList = Repository.renderableNpcs
lobbyList!!.map{ PacketRepository.send(ClearMinimapFlag::class.java, PlayerContext(it)) }
var playerTickStart = System.currentTimeMillis()
renderablePlayers.forEach(Player::tick)
Grafana.playerTickTime = (System.currentTimeMillis() - playerTickStart).toInt()
var npcTickStart = System.currentTimeMillis()
npcList!!.forEach(NPC::tick)
Grafana.npcTickTime = (System.currentTimeMillis() - npcTickStart).toInt()
}
/**
* Runs the updating part of the sequence.
*/
fun run() {
var playerRenderStart = System.currentTimeMillis()
renderablePlayers.forEach(Player::update)
Grafana.playerRenderTime = (System.currentTimeMillis() - playerRenderStart).toInt()
}
/**
@@ -72,4 +81,4 @@ class UpdateSequence
val renderablePlayers = InitializingNodeList<Player>()
}
}
}
@@ -0,0 +1,158 @@
package core.integrations.grafana
import core.ServerConstants
import core.api.StartupListener
import core.game.bots.AIRepository
import core.integrations.sqlite.SQLiteProvider
import org.json.simple.JSONObject
class Grafana : StartupListener {
override fun startup() {
if (!ServerConstants.GRAFANA_LOGGING) return
db = SQLiteProvider (ServerConstants.GRAFANA_PATH + "grafana.db", expectedTables)
db.initTables()
db.pruneOldData(ServerConstants.GRAFANA_TTL_DAYS)
}
data class GrafanaData (
val playerTickTime: Int,
val npcTickTime: Int,
val playerRenderTime: Int,
val totalTickTime: Int,
val packetProcessTime: Int,
val botPulseTime: Int,
val otherPulseTime: Int,
val pulseTimes: Array<Map.Entry<String, Int>>,
val pulseCounts: Array<Map.Entry<String, Int>>
)
companion object {
var playerTickTime = 0
var npcTickTime = 0
var playerRenderTime = 0
var totalTickTime = 0
var packetProcessTime = 0
var botPulseTime = 0
var otherPulseTime = 0
lateinit var db: SQLiteProvider
var pulseTimes = HashMap<String, Int>()
var pulseCounts = HashMap<String, Int>()
fun addPulseLength(pulseName: String, time: Int) {
if (!this::db.isInitialized) return
pulseTimes[pulseName] = (pulseTimes[pulseName] ?: 0) + time
}
fun countPulse(pulseName: String) {
if (!this::db.isInitialized) return
pulseCounts[pulseName] = (pulseCounts[pulseName] ?: 0) + 1
}
fun startTick() {
if (!this::db.isInitialized) return
playerTickTime = 0
npcTickTime = 0
playerRenderTime = 0
totalTickTime = 0
packetProcessTime = 0
botPulseTime = 0
otherPulseTime = 0
pulseTimes.clear()
pulseCounts.clear()
}
fun endTick() {
if (!this::db.isInitialized) return
val cycleData = GrafanaData (
playerTickTime,
npcTickTime,
playerRenderTime,
totalTickTime,
packetProcessTime,
botPulseTime,
otherPulseTime,
pulseTimes.entries.toTypedArray(),
pulseCounts.entries.toTypedArray()
)
db.runAsync { it ->
with (it.prepareStatement(INSERT_TOP_PULSES)) {
val topSorted = cycleData.pulseTimes.sortedByDescending { entry -> entry.value }
val rootObj = JSONObject()
val contentObj = JSONObject()
for (i in 0 until 5) {
contentObj [topSorted[i].key] = topSorted[i].value
}
rootObj["pulses"] = contentObj
setString(1, rootObj.toJSONString())
setInt(2, getNowTime())
execute()
}
with (it.prepareStatement(INSERT_PULSE_COUNT)) {
val topSorted = cycleData.pulseCounts.sortedByDescending { entry -> entry.value }
val rootObj = JSONObject()
val contentObj = JSONObject()
for (i in 0 until 5) {
contentObj [topSorted[i].key] = topSorted[i].value
}
rootObj["pulses"] = contentObj
setString(1, rootObj.toJSONString())
setInt(2, getNowTime())
execute()
}
with (it.prepareStatement(INSERT_BOT_COUNT)) {
setInt(1, AIRepository.PulseRepository.size)
setInt(2, getNowTime())
execute()
}
with (it.prepareStatement(INSERT_TICK_MEAS)) {
setInt(1, cycleData.botPulseTime)
setInt(2, cycleData.otherPulseTime)
setInt(3, cycleData.npcTickTime)
setInt(4, cycleData.playerTickTime)
setInt(5, cycleData.playerRenderTime)
setInt(6, cycleData.packetProcessTime)
setInt(7, cycleData.totalTickTime - (cycleData.botPulseTime + cycleData.otherPulseTime + cycleData.npcTickTime + cycleData.playerTickTime + cycleData.playerRenderTime + cycleData.packetProcessTime))
setInt(8, getNowTime())
execute()
}
}
}
private fun getNowTime () : Int {
return (System.currentTimeMillis() / 1000L).toInt()
}
private const val INSERT_TOP_PULSES = "INSERT INTO top_pulses (pulse_info, ts) VALUES (?, ?);"
private const val INSERT_PULSE_COUNT = "INSERT INTO high_volume_pulses (pulse_info, ts) VALUES (?, ?);"
private const val INSERT_BOT_COUNT = "INSERT INTO bot_counts (count, ts) VALUES (?, ?);"
private const val INSERT_TICK_MEAS = "INSERT INTO tick_lengths (bot_pulses, misc_pulses, npc_tick, player_tick, player_render, packet_incoming, other, ts) VALUES (?,?,?,?,?,?,?,?);"
private var expectedTables = hashMapOf(
"bot_counts" to "CREATE TABLE \"bot_counts\" (\n" +
"\t\"count\"\tINTEGER,\n" +
"\t\"ts\"\tINTEGER\n" +
")",
"high_volume_pulses" to "CREATE TABLE \"high_volume_pulses\" (\n" +
"\t\"pulse_info\"\tTEXT,\n" +
"\t\"ts\"\tINTEGER\n" +
")",
"tick_lengths" to "CREATE TABLE \"tick_lengths\" (\n" +
"\t\"bot_pulses\"\tINTEGER,\n" +
"\t\"misc_pulses\"\tINTEGER,\n" +
"\t\"npc_tick\"\tINTEGER,\n" +
"\t\"player_tick\"\tINTEGER,\n" +
"\t\"player_render\"\tINTEGER,\n" +
"\t\"packet_incoming\"\tINTEGER,\n" +
"\t\"other\"\tINTEGER,\n" +
"\t\"ts\"\tINTEGER\n" +
")",
"top_pulses" to "CREATE TABLE \"top_pulses\" (\n" +
"\t\"pulse_info\"\tTEXT,\n" +
"\t\"ts\"\tINTEGER\n" +
")"
)
}
}
@@ -0,0 +1,94 @@
package core.integrations.sqlite
import core.api.log
import core.tools.Log
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import org.sqlite.SQLiteDataSource
import java.io.File
import java.sql.Connection
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
class SQLiteProvider (private val path: String, private val expectedTables: HashMap<String,String>? = null) {
private var connection: Connection? = null
private var connectionRefs = 0
private var obtainConnectionLock = ReentrantLock()
private var dbRunLock = ReentrantLock()
private val coroutineScope = CoroutineScope(Dispatchers.IO)
private lateinit var sqlitePath: String
fun initTables(tableCreatedCallback: ((String) -> Unit)? = null) {
val pathTokens = File(path).absolutePath.split(File.separator)
val file = pathTokens.last()
val parentDir = pathTokens.dropLast(1).joinToString(File.separator)
if (!File(parentDir).exists())
File(parentDir).mkdirs()
sqlitePath = parentDir + File.separator + file
if (expectedTables?.isEmpty() != false) return
run {
with(it.prepareStatement(TABLE_CHECK)) {
for ((table, create) in expectedTables) {
setString(1, table)
val res = executeQuery()
if (!res.next())
with(it.createStatement()) { execute(create); tableCreatedCallback?.invoke(table) }
}
}
}
}
fun pruneOldData (daysToKeep: Int, timestampFieldName: String = "ts") {
if (expectedTables?.isEmpty() != false) return
val timeDiff = daysToKeep * 24 * 60 * 60
val nowTime = System.currentTimeMillis() / 1000
run {
with (it.createStatement()) {
for ((table, _) in expectedTables) {
execute("DELETE FROM $table WHERE $timestampFieldName <= ${nowTime - timeDiff};")
}
}
}
}
fun runAsync (closure: (conn: Connection) -> Unit) : Job {
return coroutineScope.launch { run (closure) }
}
fun run (closure: (conn: Connection) -> Unit) {
dbRunLock.tryLock(10000L, TimeUnit.MILLISECONDS)
connectionRefs++
val con = connect()
closure.invoke(con)
connectionRefs--
if(connectionRefs == 0) {
con.close()
}
dbRunLock.unlock()
}
private fun connect(): Connection
{
obtainConnectionLock.tryLock(10000L, TimeUnit.MILLISECONDS)
if (connection == null || connection!!.isClosed)
{
val ds = SQLiteDataSource()
ds.url = "jdbc:sqlite:$sqlitePath"
connection = ds.connection
}
obtainConnectionLock.unlock()
return connection!!
}
companion object {
const val TABLE_CHECK = "SELECT name FROM sqlite_master WHERE type='table' AND name=?;"
}
}
@@ -19,6 +19,7 @@ import java.lang.Long.max
import java.text.SimpleDateFormat
import java.util.*
import kotlin.system.exitProcess
import core.integrations.grafana.*
/**
* Handles the running of pulses and writing of masks, etc
@@ -34,6 +35,7 @@ class MajorUpdateWorker {
started = true
Thread.sleep(600L)
while (running) {
Grafana.startTick()
val start = System.currentTimeMillis()
Server.heartbeat()
handleTickActions()
@@ -77,6 +79,8 @@ class MajorUpdateWorker {
}
val end = System.currentTimeMillis()
Grafana.totalTickTime = (end - start).toInt()
Grafana.endTick()
/* ServerMonitor.eventQueue.add(GuiEvent.UpdateTickTime(end - start))
ServerMonitor.eventQueue.add(GuiEvent.UpdatePulseCount(GameWorld.Pulser.TASKS.size))*/
Thread.sleep(max(600 - (end - start), 0))
@@ -87,7 +91,9 @@ class MajorUpdateWorker {
fun handleTickActions(skipPulseUpdate: Boolean = false) {
try {
var packetStart = System.currentTimeMillis()
PacketProcessor.processQueue()
Grafana.packetProcessTime = (System.currentTimeMillis() - packetStart).toInt()
//disconnect all players waiting to be disconnected
Repository.disconnectionQueue.update()
@@ -127,4 +133,4 @@ class MajorUpdateWorker {
running = false
worker.interrupt()
}
}
}
+9
View File
@@ -31,6 +31,15 @@ database_password = ""
database_address = "127.0.0.1"
database_port = "3306"
[integrations]
grafana_logging = false
grafana_log_path = "@data/logs"
#how long grafana will keep old data for in days (anything older is pruned once on startup)
grafana_log_ttl_days = 7
#discord_ge_webhook = "webhook link"
#discord_moderation_webhook = "webhook link"
#openrsc_integration_webhook = "webhook link"
[world]
name = "2009scape"