yuriy/2009scape
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Grafana Integration + Unified SQLite Boilerplate + Fixed bot movement performance regression

Browse Source
This commit is contained in:
Ceikry
2023-07-31 14:59:53 +00:00
parent ac0da92a58
commit ee79b0750c
13 changed files with 984 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Server/src/main/core/game/bots/GeneralBotCreator.kt
+33 -6
View File
@@ -1,12 +1,21 @@
package core.game.bots
import content.global.bots.Idler
import core.Server
import core.game.node.entity.player.Player
import core.game.system.task.Pulse
import core.game.world.GameWorld
import core.game.world.map.Location
import core.tools.RandomFunction
import core.Server
import content.global.bots.Idler
import core.api.*
import core.game.interaction.MovementPulse
import core.tools.Log
class GBCTick : TickListener {
override fun tick() {
GeneralBotCreator.botPulsesTriggeredThisTick = 0
}
}
class GeneralBotCreator {
//org/crandor/net/packet/in/InteractionPacket.java <<< This is a very useful class for learning to code bots
@@ -28,18 +37,34 @@ class GeneralBotCreator {
player.setAttribute("botting:script",pulse)
}
inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1){
companion object {
var botPulsesTriggeredThisTick = 0
}
inner class BotScriptPulse(public val botScript: Script, val isPlayer: Boolean = false) : Pulse(1) {
var ticks = 0
init {
botScript.init(isPlayer)
}
var randomDelay = 0
var lastBotLocation: Location = botScript.bot.location.transform(0,0,0)
var lastBotMoveTicks = getWorldTicks()
override fun pulse(): Boolean {
if(randomDelay > 0){
randomDelay -= 1
return false
}
if (botScript.bot.pulseManager.hasPulseRunning()) {
if (botScript.bot.pulseManager.current is MovementPulse) {
if (botScript.bot.location != lastBotLocation) {
lastBotLocation = botScript.bot.location.transform(0,0,0)
lastBotMoveTicks = getWorldTicks()
}
if (lastBotLocation == botScript.bot.location && getWorldTicks() - lastBotMoveTicks > 5) {
botScript.bot.pulseManager.current.stop()
}
}
}
/*
* Chatboxes and interfaces will cause the authentic interaction subsystem
@@ -73,10 +98,12 @@ class GeneralBotCreator {
}*/
if(!botScript.running) return true //has to be separated this way or it double-submits the respawn pulse.
if (botPulsesTriggeredThisTick++ >= 50)
return false
val idleRoll = RandomFunction.random(50)
val idleRoll = RandomFunction.random(10)
if(idleRoll == 2 && botScript !is Idler){
randomDelay += RandomFunction.random(2,20)
randomDelay += RandomFunction.random(20,50)
return false
}
botScript.tick()
Server/src/main/core/game/ge/GEDB.kt
+50 -147
View File
@@ -1,71 +1,33 @@
package core.game.ge
import core.ServerConstants
import core.cache.def.impl.ItemDefinition
import core.integrations.sqlite.SQLiteProvider
import org.json.simple.JSONArray
import org.json.simple.JSONObject
import org.json.simple.parser.JSONParser
import org.sqlite.SQLiteDataSource
import core.ServerConstants
import java.io.File
import java.io.FileReader
import java.sql.Connection
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
/**
* Collection of methods for interacting with the grand exchange databases
* @author Ceikry
*/
object GEDB {
private var pathString = ""
public var connection: Connection? = null
private var initialized = false
private var connectionRefs = 0
private var obtainConnectionLock = ReentrantLock()
private var dbRunLock = ReentrantLock()
lateinit var db: SQLiteProvider
fun init() {
init(File(ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db").absolutePath)
init (ServerConstants.GRAND_EXCHANGE_DATA_PATH + "grandexchange.db")
}
//This needs to be a separate method, so we can call it after the server config has been parsed
fun init(path: String)
{
if(initialized) return
pathString = path
//Check if the grandexchange.db file already exists. If not, create it and create the tables.
if(!File(path).exists())
generateAndTransfer()
fun init (path: String) {
db = SQLiteProvider(path, expectedTables)
db.initTables { createdTable: String ->
if (createdTable == "price_index")
populateInitialPriceIndex()
}
}
@JvmStatic fun run(closure: (conn: Connection) -> Unit) {
dbRunLock.tryLock(10000L, TimeUnit.MILLISECONDS)
connectionRefs++
val con = connect()
closure.invoke(con)
connectionRefs--
if(connectionRefs == 0) {
con.close()
}
dbRunLock.unlock()
}
private fun connect(): Connection
{
obtainConnectionLock.tryLock(10000L, TimeUnit.MILLISECONDS)
if (connection == null || connection!!.isClosed)
{
val ds = SQLiteDataSource()
ds.url = "jdbc:sqlite:$pathString"
connection = ds.connection
}
obtainConnectionLock.unlock()
return connection!!
db.run(closure)
}
private fun convertJsonArray(arr: JSONArray): String
@@ -85,107 +47,48 @@ object GEDB {
return sb.toString()
}
private fun generateAndTransfer()
var expectedTables = hashMapOf(
"player_offers" to "CREATE TABLE player_offers(" +
"uid INTEGER PRIMARY KEY ASC," +
"player_uid INTEGER," +
"item_id INTEGER," +
"amount_total INTEGER," +
"amount_complete INTEGER," +
"offered_value INTEGER," +
"time_stamp INTEGER," +
"offer_state INTEGER," +
"is_sale INTEGER," +
"withdraw_items STRING ," +
"slot_index INTEGER," +
"total_coin_xc INTEGER)",
"bot_offers" to "CREATE TABLE bot_offers(" +
"item_id INTEGER," +
"amount INTEGER)",
"price_index" to "CREATE TABLE price_index(" +
"item_id INTEGER," +
"value INTEGER," +
"total_value INTEGER," +
"unique_trades INTEGER," +
"last_update INTEGER)"
)
private fun populateInitialPriceIndex()
{
val conn = connect() //for sqlite jdbc, attempting to connect to a non-existing .db file creates it.
val statement = conn.createStatement()
//table for tracking player offers - replaces offer_dispatch.json
statement.execute(
"CREATE TABLE player_offers(" +
"uid INTEGER PRIMARY KEY ASC," +
"player_uid INTEGER," +
"item_id INTEGER," +
"amount_total INTEGER," +
"amount_complete INTEGER," +
"offered_value INTEGER," +
"time_stamp INTEGER," +
"offer_state INTEGER," +
"is_sale INTEGER," +
"withdraw_items STRING ," +
"slot_index INTEGER," +
"total_coin_xc INTEGER)"
)
//table for tracking bot offers - replaces bot_offers.json
statement.execute(
"CREATE TABLE bot_offers(" +
"item_id INTEGER," +
"amount INTEGER)"
)
//table for tracking price index data - replaces gedb.xml
statement.execute(
"CREATE TABLE price_index(" +
"item_id INTEGER," +
"value INTEGER," +
"total_value INTEGER," +
"unique_trades INTEGER," +
"last_update INTEGER)"
)
//check if the old .json and .xml files exist, and if they do, read them into the sqlite database and remove them.
val playerJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "offer_dispatch.json"
val botJson = ServerConstants.GRAND_EXCHANGE_DATA_PATH + "bot_offers.json"
if(File(playerJson).exists())
{
val parser = JSONParser()
val reader = FileReader(playerJson)
val saveFile = parser.parse(reader) as JSONObject
if (saveFile.containsKey("offers")) {
val offers = saveFile["offers"] as JSONArray
for (offer in offers) {
val o = offer as JSONObject
statement.execute("insert into player_offers(player_uid,item_id,amount_total,amount_complete,offered_value,time_stamp,offer_state,is_sale,withdraw_items,total_coin_xc,slot_index) " +
"values(" +
"${o["playerUID"]}," +
"${o["itemId"]}," +
"${o["amount"]}," +
"${o["completedAmount"]}," +
"${o["offeredValue"]}," +
"${o["timeStamp"]}," +
"${o["offerState"]}," +
"${if (o["sale"] as Boolean) 1 else 0}," +
"'" + convertJsonArray(o["withdrawItems"] as JSONArray) + "'," +
"${o["totalCoinExchange"]}," +
"-1)"
)
}
}
reader.close()
//File(playerJson).delete()
}
if(File(botJson).exists())
{
val parser = JSONParser()
val reader = FileReader(botJson)
val saveFile = parser.parse(reader) as JSONObject
if (saveFile.containsKey("offers")) {
val offers = saveFile["offers"] as JSONArray
for (offer in offers) {
val o = offer as JSONObject
statement.execute("insert into bot_offers " +
"values(${o["item"]},${o["qty"]})")
}
}
reader.close()
}
statement.close()
//price index isn't worth transferring, so we're just going to make a new one.
ItemDefinition.getDefinitions().values.forEach { def ->
if(def.isTradeable){
PriceIndex.allowItem(def.id)
run {
with (it.prepareStatement(PriceIndex.INSERT_QUERY)) {
ItemDefinition.getDefinitions().values.forEach { def ->
if(def.isTradeable) {
setInt(1, def.id)
setInt(2, def.getAlchemyValue(true))
setInt(3, def.getAlchemyValue(true))
setInt(4, 0)
setInt(5, 0)
execute()
}
}
}
}
}
}
Server/src/main/core/game/interaction/MovementPulse.java
+5 -1
View File
@@ -192,6 +192,8 @@ public abstract class MovementPulse extends Pulse {
if (!mover.getViewport().getRegion().isActive())
return false;
if (!isRunning()) return true;
if (!validate()) {
stop();
return true;
@@ -295,6 +297,8 @@ public abstract class MovementPulse extends Pulse {
if (destination instanceof Entity || interactLocation == null || (!mover.getWalkingQueue().hasPath() && interactLocation.getDistance(mover.getLocation()) > 0) || (usingTruncatedPath && destination.getLocation().getDistance(mover.getLocation()) < 14)) {
if (!checkAllowMovement())
return;
if (destination instanceof Entity && previousLoc != null && previousLoc.equals(loc) && mover.getWalkingQueue().hasPath())
return;
Path path;
Pair<Boolean, Location> truncation = truncateLoc(mover, loc != null ? loc : destination.getLocation());
@@ -401,7 +405,7 @@ public abstract class MovementPulse extends Pulse {
Vector pathDiff = Vector.betweenLocs (destination.getLocation(), Location.create(p.getX(), p.getY(), plane));
Location predictedCenterPos = (destination.getMathematicalCenter().plus(pathDiff)).toLocation(plane);
Vector toPlayerNormalized = Vector.betweenLocs(predictedCenterPos, mover.getCenterLocation()).normalized();
return predictedCenterPos.transform(toPlayerNormalized.times(destination.size()));
return predictedCenterPos.transform(toPlayerNormalized.times(destination.size() + 1));
}
Server/src/main/core/game/node/entity/player/info/PlayerMonitor.kt
+43 -63
View File
@@ -1,16 +1,15 @@
package core.game.node.entity.player.info
import core.ServerConstants
import core.api.getItemName
import core.game.container.Container
import core.game.node.entity.player.Player
import core.game.node.entity.skill.Skills
import core.integrations.discord.Discord
import kotlinx.coroutines.*
import org.sqlite.SQLiteDataSource
import core.ServerConstants
import core.integrations.sqlite.SQLiteProvider
import core.tools.Log
import core.tools.SystemLogger
import java.io.File
import kotlinx.coroutines.Job
import kotlinx.coroutines.cancel
import java.sql.Connection
import java.util.concurrent.LinkedBlockingQueue
import kotlin.math.abs
@@ -18,6 +17,25 @@ import kotlin.math.abs
object PlayerMonitor {
private val eventQueue = LinkedBlockingQueue<LogEvent>()
private var activeTask: Job? = null
private lateinit var db: SQLiteProvider
var expectedTables = hashMapOf(
"chat_logs" to """
CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC );
""",
"misc_logs" to """
CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC);
""",
"trade_logs" to """
CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC );
""",
"xp_gains" to """
CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC)
""",
"wealth_logs" to """
CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC )
"""
)
@JvmStatic fun logWealthChange(player: Player, totalWealth: Long, diff: Long) {
val event = LogEvent.WealthLog(
@@ -120,72 +138,34 @@ object PlayerMonitor {
}
}
fun processQueuedEvents() {
val path = ServerConstants.LOGS_PATH + "playerlogs.db"
if (activeTask?.isActive == true)
return
if (!File(path).exists()) {
createSqliteDatabase(path)
}
activeTask = GlobalScope.launch {
val conn = connect(path)
conn.use {
while (eventQueue.isNotEmpty()) {
val event = withContext(Dispatchers.IO) { eventQueue.take() }
process(event, it)
}
}
fun init () {
if (!this::db.isInitialized) {
var path = ServerConstants.LOGS_PATH + "playerlogs.db"
db = SQLiteProvider(path, expectedTables)
db.initTables()
}
}
@JvmStatic fun flushRemainingEventsImmediately() {
core.api.log(this::class.java, Log.FINE, "Flushing player log events...")
val path = ServerConstants.LOGS_PATH + "playerlogs.db"
if (!File(path).exists()) {
createSqliteDatabase(path)
}
if (activeTask != null)
activeTask?.cancel("Interrupted by shutdown. This is probably fine.")
val conn = connect(path)
conn.use {
fun processQueuedEvents() {
init()
if (activeTask?.isActive == true)
return
activeTask = db.runAsync {
while (eventQueue.isNotEmpty())
process(eventQueue.take(), it)
}
}
private fun connect(path: String) : Connection {
val source = SQLiteDataSource()
source.url = "jdbc:sqlite:$path"
return source.connection
}
private fun createSqliteDatabase(path: String) {
if (!File(ServerConstants.LOGS_PATH).exists())
File(ServerConstants.LOGS_PATH).mkdirs()
val connection = connect(path)
val stmt = connection.createStatement()
stmt.execute(
"""
CREATE TABLE "chat_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "message" TEXT, "timestamp" NUMERIC );
""")
stmt.execute(
"""
CREATE TABLE "misc_logs" ( "player" TEXT, "uid" INTEGER, "type" TEXT, "details" TEXT , "timestamp" NUMERIC);
""")
stmt.execute(
"""
CREATE TABLE "trade_logs" ( "player_a" TEXT, "player_b" TEXT, "uid_a" INTEGER, "uid_b" INTEGER, "items_a" TEXT, "items_b" TEXT, "timestamp" NUMERIC );
""")
stmt.execute(
"""
CREATE TABLE "xp_gains" ( "player" TEXT, "uid" INTEGER, "attack" INTEGER, "defence" INTEGER, "strength" INTEGER, "hitpoints" INTEGER, "ranged" INTEGER, "prayer" INTEGER, "magic" INTEGER, "cooking" INTEGER, "woodcutting" INTEGER, "fletching" INTEGER, "fishing" INTEGER, "firemaking" INTEGER, "crafting" INTEGER, "smithing" INTEGER, "mining" INTEGER, "herblore" INTEGER, "agility" INTEGER, "thieving" INTEGER, "slayer" INTEGER, "farming" INTEGER, "runecrafting" INTEGER, "hunter" INTEGER, "construction" INTEGER, "summoning" INTEGER , "timestamp" NUMERIC)
""")
stmt.execute(
"""
CREATE TABLE "wealth_logs" ( "player" TEXT, "uid" INTEGER, "total" NUMERIC, "diff" NUMERIC, "timestamp" NUMERIC )
""")
stmt.close()
connection.close()
@JvmStatic fun flushRemainingEventsImmediately() {
core.api.log(this::class.java, Log.FINE, "Flushing player log events...")
init()
if (activeTask != null)
activeTask?.cancel("Interrupted by shutdown. This is probably fine.")
db.run {
while (eventQueue.isNotEmpty())
process(eventQueue.take(), it)
}
}
private fun process(event: LogEvent, conn: Connection) {
Server/src/main/core/game/system/config/ServerConfigParser.kt
+7 -4
View File
@@ -139,16 +139,16 @@ object ServerConfigParser {
ServerConstants.BANK_BOOTH_QUICK_OPEN = data.getBoolean("world.bank_booth_quick_open", false)
ServerConstants.BANK_BOOTH_NOTE_ENABLED = data.getBoolean("world.bank_booth_note_enabled", true)
ServerConstants.BANK_BOOTH_NOTE_UIM = data.getBoolean("world.bank_booth_note_uim", true)
ServerConstants.DISCORD_GE_WEBHOOK = data.getString("server.discord_webhook", "")
ServerConstants.DISCORD_GE_WEBHOOK = data.getString("integrations.discord_ge_webhook", "")
ServerConstants.WATCHDOG_ENABLED = data.getBoolean("server.watchdog_enabled", true)
ServerConstants.I_AM_A_CHEATER = data.getBoolean("world.i_want_to_cheat", false)
ServerConstants.USE_AUTH = data.getBoolean("server.use_auth", true)
ServerConstants.PERSIST_ACCOUNTS = data.getBoolean("server.persist_accounts", true)
ServerConstants.DAILY_ACCOUNT_LIMIT = data.getLong("server.daily_accounts_per_ip", 3L).toInt()
ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("server.moderation_webhook", "")
ServerConstants.DISCORD_MOD_WEBHOOK = data.getString("integrations.discord_moderation_webhook", "")
ServerConstants.NOAUTH_DEFAULT_ADMIN = data.getBoolean("server.noauth_default_admin", false)
ServerConstants.DRAGON_AXE_USE_OSRS_SPEC = data.getBoolean("world.dragon_axe_use_osrs_spec", false)
ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("server.openrsc_integration_webhook", "")
ServerConstants.DISCORD_OPENRSC_HOOK = data.getString("integrations.openrsc_integration_webhook", "")
ServerConstants.ENABLE_GLOBALCHAT = data.getBoolean("world.enable_globalchat", true)
ServerConstants.MAX_PATHFIND_DISTANCE = data.getLong("server.max_pathfind_dist", 25L).toInt()
ServerConstants.IRONMAN_ICONS = data.getBoolean("world.ironman_icons", false)
@@ -156,6 +156,9 @@ object ServerConfigParser {
ServerConstants.PLAYER_STOCK_RECIRCULATE = data.getBoolean("world.playerstock_bot_offers", true)
ServerConstants.BOTSTOCK_LIMIT = data.getLong("world.botstock_limit", 5000L).toInt()
ServerConstants.BETTER_AGILITY_PYRAMID_GP = data.getBoolean("world.better_agility_pyramid_gp", true)
ServerConstants.GRAFANA_PATH = data.getPath("integrations.grafana_log_path")
ServerConstants.GRAFANA_LOGGING = data.getBoolean("integrations.grafana_logging", false)
ServerConstants.GRAFANA_TTL_DAYS = data.getLong("integrations.grafana_log_ttl_days", 7L).toInt()
val logLevel = data.getString("server.log_level", "VERBOSE").uppercase()
ServerConstants.LOG_LEVEL = parseEnumEntry<LogLevel>(logLevel) ?: LogLevel.VERBOSE
@@ -164,7 +167,7 @@ object ServerConfigParser {
private fun Toml.getPath(key: String): String{
try {
return parsePath(getString(key).replace("@data", ServerConstants.DATA_PATH!!))
return parsePath(getString(key, "@data/").replace("@data", ServerConstants.DATA_PATH!!))
} catch (e: Exception){
log(this::class.java, Log.ERR, "Error parsing key: $key")
exitProcess(0)
Server/src/main/core/game/world/PulseRunner.kt
+20
View File
@@ -5,7 +5,9 @@ import core.game.system.task.Pulse
import core.game.bots.GeneralBotCreator
import core.tools.Log
import core.tools.SystemLogger
import core.ServerConstants
import java.util.concurrent.LinkedBlockingQueue
import core.integrations.grafana.*
class PulseRunner {
private val pulses = LinkedBlockingQueue<Pulse>()
@@ -19,6 +21,8 @@ class PulseRunner {
fun updateAll() {
val pulseCount = pulses.size
var totalTimeBotPulses = 0
var totalTimeOtherPulses = 0
for (i in 0 until pulseCount) {
val pulse = pulses.take()
@@ -33,8 +37,24 @@ class PulseRunner {
}
}
var pulseName = pulse::class.java.name
if (pulse is GeneralBotCreator.BotScriptPulse || pulseName.contains("ScriptAPI")) {
totalTimeBotPulses += elapsedTime.toInt()
} else {
totalTimeOtherPulses += elapsedTime.toInt()
}
Grafana.addPulseLength (pulseName, elapsedTime.toInt())
Grafana.countPulse (pulseName)
notifyIfTooLong(pulse, elapsedTime)
}
if (ServerConstants.GRAFANA_LOGGING) {
Grafana.botPulseTime = totalTimeBotPulses
Grafana.otherPulseTime = totalTimeOtherPulses
}
}
private fun measure(logic: () -> Unit): Long {
Server/src/main/core/game/world/update/UpdateSequence.kt
+10 -1
View File
@@ -9,6 +9,7 @@ import core.net.packet.PacketRepository
import core.net.packet.context.PlayerContext
import core.net.packet.out.ClearMinimapFlag
import core.game.world.repository.Repository
import core.integrations.grafana.*
/**
* The entity update sequence.
@@ -32,15 +33,23 @@ class UpdateSequence
playersList = renderablePlayers
npcList = Repository.renderableNpcs
lobbyList!!.map{ PacketRepository.send(ClearMinimapFlag::class.java, PlayerContext(it)) }
var playerTickStart = System.currentTimeMillis()
renderablePlayers.forEach(Player::tick)
Grafana.playerTickTime = (System.currentTimeMillis() - playerTickStart).toInt()
var npcTickStart = System.currentTimeMillis()
npcList!!.forEach(NPC::tick)
Grafana.npcTickTime = (System.currentTimeMillis() - npcTickStart).toInt()
}
/**
* Runs the updating part of the sequence.
*/
fun run() {
var playerRenderStart = System.currentTimeMillis()
renderablePlayers.forEach(Player::update)
Grafana.playerRenderTime = (System.currentTimeMillis() - playerRenderStart).toInt()
}
/**
@@ -72,4 +81,4 @@ class UpdateSequence
val renderablePlayers = InitializingNodeList<Player>()
}
}
}