#!/usr/bin/env python3 # -*- coding: utf-8 -*- # coding: utf8 import mmap import traceback import argparse from argparse import Namespace import re import sys from glob import glob from datetime import datetime, timedelta from typing import Any from functools import wraps from time import time as ti from tqdm import tqdm from ffxiv_aku import * from ffxiv_aku import get_any_Logdata, storeFilesInTmp, loadDataTheQuickestWay, getPlayerActionIDs, getPlayerStatusIDs, get_list_of_all_pet_names, splitLogLineByVersion, pretty_json from ffxiv_aku import print_color_green, print_color_yellow, print_color_red, print_pretty_json, convertLogdataFromClassToJson, getLogdataAsClassNew from dateutil import parser as dp def timing(f): @wraps(f) def wrap(*args, **kw): ts = ti() result = f(*args, **kw) te = ti() print(f'func:{f.__name__} took: {te-ts:.4f} sec') return result return wrap # language to safe results lang = "de" # language of the logfile itself e.g. from Azubi set it to en compare_lang = "de" example_lines: dict[Any, Any] = {} result: dict[Any, Any] = {} class SETTINGS: exclude_stats_by_version = 22106 storeFilesInTmp(False) ttype: dict[str, dict[str, str]] = loadDataTheQuickestWay("TerritoryType.json") cfc: dict[str, dict[str, str]] = loadDataTheQuickestWay("ContentFinderCondition.json") cj: dict[str, dict[str, str]] = loadDataTheQuickestWay("ClassJob.json") action: dict[str, dict[str, str]] = loadDataTheQuickestWay("Action.json") bnpcname: dict[str, dict[str, str]] = loadDataTheQuickestWay("BNpcName.json") status: dict[str, dict[str, str]] = loadDataTheQuickestWay("Status.json") placename: dict[str, dict[str, str]] = loadDataTheQuickestWay("PlaceName.json") npcyell: dict[str, Any] = loadDataTheQuickestWay("NpcYell.json") instancecontenttextdata: dict[str, Any] = loadDataTheQuickestWay("InstanceContentTextData.json") fateevent: dict[str, Any] = loadDataTheQuickestWay("FateEvent.json") bnpc_names = [ x['Singular_de'] for k, x in bnpcname.items() ] #getEnemyChatLogLines #fates: dict[str, dict[str, str]] = loadDataTheQuickestWay("fate_all.json", translate=True) LANGUAGES:dict[str, str] = { "English": "en", "German": "de", "French": "fr", "Japanese": "ja", "3": "de", "1": "en" } def getLanguageToImport(line: str) -> str | None: m: Any = re.search(r'(?:Language ID: )([A-Za-z0-9]+)', line) if not m: return None language_code_int: Any = m.group(1) # this filters for only german language #if language_code_int not in ['3', 'German']: # return None if LANGUAGES.get(language_code_int, None): return language_code_int return None ignore_skill: list[str] = ['322'] pActionIDs: list[str] = getPlayerActionIDs() ignore_skills_on_empty_enemies: list[str] = [x.upper() for x in pActionIDs] g_ignore_status: list[str] = getPlayerStatusIDs() excluded_enemies: list[str] = get_list_of_all_pet_names() excluded_enemies: list[str] = [e.lower() for e in excluded_enemies] filter_player_status: list[str] = g_ignore_status + ["864", "866", "92c", "A07"] def get_num_lines(file_path: str) -> int: fp = open(file_path, "r+") buf = mmap.mmap(fp.fileno(), 0) lines = 0 while buf.readline(): lines += 1 return lines def check_for_act_restart(line: str, config: dict[str, str | bool | None | int]) -> dict[str, str | bool | None | int]: if line.startswith("FFXIV PLUGIN VERSION: "): config['location'] = "_Unknown" config['location_id'] = "_Unknown" config['location_name'] = "" config['is_bluemage'] = False config['is_baldesion_active'] = False config['is_castrum_active'] = False config['is_dalriada_active'] = False config['is_kreszentiaturmdesblutes_active'] = False config['saved_location_when_entering_special_zone'] = "" config['found_correct_language'] = False config['switchedLanguageCode'] = False config['placename_locked'] = None config["player"] = "" # print("\nFound new start of ACT, Clear Log now!", end="") return config def getInstanceFromTerritoryType(c_id: dict[str, str]) -> str: for _id, content in cfc.items(): if content["TerritoryType"].get("Name_de", "") == c_id["Name"]: return content[f"Name_{lang}"].replace("<", "").replace(">", "").replace("\u00ad", "") #print_color_red("Could not find Instance, use prewrote unreals in [getInstanceFromTerritoryType]") if c_id["Name"] == "r1fc_3": return "Traumprüfung Shiva" if c_id["Name"] == "s1fa_4": return "Traumprüfung Titan" if c_id["Name"] == "a2fd_3": return "Traumprüfung Sephiroth" if c_id["Name"] == "a2fe_3": return "Traumprüfung Sophia" #print_color_red("Error at getInstace: " + str(c_id)) return c_id['PlaceName'][f"Name_{lang}"].replace("\u00ad", "") def getLocationName(_id: str, location: str) -> str: try: entry = ttype[str(int(_id, 16))] value = getInstanceFromTerritoryType(entry).replace("\u00ad", "").replace("(Episch)", "(episch)").replace("(Schwer)", "(schwer)").replace("(Extrem)", "(extrem)").replace("(Fatal)", "(fatal)") except KeyError: traceback.print_exc() value = location.replace("(Episch)", "(episch)").replace("(Schwer)", "(schwer)").replace("(Extrem)", "(extrem)").replace("(Fatal)", "(fatal)") return value def getActionName(_id: str, skill_name: str) -> str: try: return fix_names(action[str(int(_id, 16))][f"Name_{lang}"].title()) except KeyError: traceback.print_exc() return skill_name.title() def getStatusName(_id: str, skill_name: str) -> str: try: return fix_names(status[str(int(_id, 16))][f"Name_{lang}"].title()) except KeyError: traceback.print_exc() return skill_name.title() def getBNPCnName(npc_name: str) -> str: if not compare_lang == lang: for x_id, npc in bnpcname.items(): if npc[f"Singular_{compare_lang}"].lower() == npc_name.lower(): return fixBNPCName(npc[f"Singular_{lang}"].title()) return fixBNPCName(npc_name.title()) return fixBNPCName(npc_name.title()) def fixBNPCName(npc_name: str) -> str: npc_name = npc_name.title() npc_name = re.sub(r" (Ii|Iii|IIi|Vi|Vii|Viii|Iv|Ix|Xi|Cxliv|Xii|Xiii|Xliii|Iii-|Vii-|Xxiv|013Bl|Xii\.)[\. |-]", lambda x: x.group(0).upper(), npc_name) npc_name = re.sub(r" (Ii|Iii|IIi|Vi|Vii|Viii|Iv|Ix|Xi|Cxliv|Xii|Xiii|Xliii|Iii-|Vii-|Xxiv|013Bl|Xii\.)$", lambda x: x.group(0).upper(), npc_name) npc_name = re.sub(r"[ |-](Ii)[ |-]", lambda x: x.group(0).upper(), npc_name) npc_name = re.sub(r"('[a-zA-Z])(?! )", lambda x: x.group(0).upper(), npc_name) return npc_name def compareBNPCnNameAndID(npc_name: str, _id: str) -> bool: if not compare_lang == lang: for x_id, npc in bnpcname.items(): if str(x_id) == (str(_id)): if npc[f"Singular_{compare_lang}"].lower() == npc_name.lower(): return True return False return False def getPlacenameName(place_name: str) -> str: if not compare_lang == lang: for x_id, npc in placename.items(): if npc[f"Name_{compare_lang}"].lower() == place_name.lower() or npc[f"Name_{compare_lang}"].lower().replace("the ", "") == place_name.lower(): return npc[f"Name_{lang}"].title() return place_name.title() return place_name.title() def getStatusIcon(_id: str) -> str: try: return status[str(int(_id, 16))]["Icon"]['path'].replace(".png", "_hr1.png") except KeyError: return "" def check_for_location(jline: dict[str, str], config: dict[str, str | bool | None | int], result: dict[str, Any]) -> tuple[dict[str, Any], dict[str, str | bool | None | int]]: config['location'] = jline['zonename'] config['location_id'] = jline['zoneid'] config['is_bluemage'] = True if config['location_id'] == "31C" else False config['placename_locked'] = None config['location_name'] = getLocationName(config['location_id'], config['location']) if result.get(config['location_name'], None) is None: result[config['location_name']] = { 'combatants': {}, "contentzoneid": [], "music": [], "zone": [] } print_color_yellow(f"\nFound new Location {config['location_name']}' - '{config['location_id']}' ({config['location']})!", end="") return config, result # blue is magic, orange is phisical in fflogs damage_type: dict[str, str] = { "-1": "WeaponOverride", "0": "None", "1": "Blunt", "2": "Piercing", "3": "Slashing", "4": "Shot", "5": "Magical", "6": "Darkness", "7": "Sound Wave", "8": "Limit Break" # Limitbreak } element_type: dict[str, str] = { "0": "None", "1": "Fire", "2": "Ice", "3": "Wind", "4": "Stone", "5": "Lightning", "6": "Water", "7": "Unaspected" } def get_damage_values(dmg: str) -> int: if len(dmg) <= 4: damage = dmg.ljust(8, "0") else: damage = dmg.zfill(8) if damage == "00000000": return 0 if damage[4:6] == "40": a = hex(int(damage[0:2], 16)).split("0x")[1].zfill(2) b = int(damage[2:4], 16) d = int(damage[6:8], 16) e = hex(b - d).split("0x")[1].zfill(2) b = hex(b).split("0x")[1].zfill(2) d = hex(d).split("0x")[1].zfill(2) c_damage = int(f"{d}{a}{e}", 16) # print_color_blue(f"{damage} > {d} {a} ({b}-{d}) > {d}{a}{e} > {c_damage}") return c_damage else: c_damage = int(damage[0:4], 16) # print_color_yellow(f"{damage} > {damage[0:4]} > {c_damage}") return c_damage def getStatus(m_id: str, name: str, _enemy_status: dict[str, str]) -> dict[str, str] | None: nm_id: str = m_id.rstrip("0") nm_id = m_id res: list[Any] = [] for key, value in status.items(): hex_id = str(hex(int(key))) if m_id.startswith(hex_id) and hex_id.startswith(nm_id): if value[f'Name_{compare_lang}'] == name: return value value['0xID'] = hex_id res.append(value) if len(res) == 1: return res[0] for value in res: if _enemy_status.get(value['0xID'], None): return value #print_color_red(f"{name} -> {m_id}") for value in res: print_color_red(f'{value["0xID"]} -> {value[f"Name_{compare_lang}"]}') def get_attack_type(line: str, _enemy_status: dict[str, str]) -> tuple[str, str, int, list[Any]]: # get element type and magic type type_of: str = damage_type["0"] element_of: str = element_type["0"] added_status_codes: list[Any] = [] damage = 0 r = line.split("|") x = r[8:24] max_counter = int(len(x) / 2) for i in range(max_counter): key = x[i * 2] value = x[i * 2 + 1] #print_color_yellow(f"{key=} -> {value=}") #print_color_yellow(line) if key == "0" and value == "0": continue elif key.lower().endswith("e"): #quisquous — Yeah, you need to read from the right and drop the two right bytes / 4 characters. #Most fields are 4 bytes with leading zeroes dropped, so 950000 is really 00950000 x_stat = getStatus(value[:-4], r[5], _enemy_status) if x_stat is not None: added_status_codes.append(x_stat['0xID']) elif len(key) > 4 and value != 0: try: element_of = key[0] type_of = key[1] element_of = element_type.get(str(element_of), "Unknown") type_of = damage_type.get(str(type_of), "Unknown") damage = get_damage_values(value) except Exception as e: type_of = damage_type["0"] element_of = element_type["0"] damage = 0 pass #print_color_blue(f"{damage=}") return type_of, element_of, damage, added_status_codes def empty(y: Any, x: Any) -> str: return "" def setupDicct(result: dict[str, Any], location_name: str) -> dict[str, Any]: # if result[location_name] is not existing, create it as empty dict if result.get(location_name, None) is None: result[location_name] = {} print(f"Created new Location: {location_name}") return result def setZoneIfPossible(result: dict[str, Any], config: dict[str, str | bool | None | int]) -> dict[str, Any]: location_name: str = config['location_name'] # type: ignore location_zone_regex: str = config['placename_locked'] # type: ignore # if a new location is found containig a Noch 15 sekunden bis sich X schliest message if location_zone_regex: if not result[location_name].get("zone", None) or isinstance(result[location_name].get("zone", None), str): result[location_name]["zone"] = [] if location_zone_regex not in result[location_name]["zone"]: result[location_name]["zone"].append(location_zone_regex) return result #TODO check that names are correct def addEntryToResults(result, config: dict[str, str | bool | None | int], data, _type, damage_type, element_type, damage, getName, added_status_codes=[]): location_name: str = config['location_name'] # type: ignore # if data[id] not in array, add it as template {} if not result[location_name][data['user_name']][_type].get(data['id'], None): result[location_name][data['user_name']][_type][data['id']] = {} if _type == "skill": # once aoe, always aoe if not result[location_name][data['user_name']][_type][data['id']].get("type_id", None) == "22": result[location_name][data['user_name']][_type][data['id']]["type_id"] = data['type'] result[location_name][data['user_name']][_type][data['id']]["name"] = getName(data['id'], data['name']) if not result[location_name][data['user_name']][_type][data['id']].get("damage_type", None): result[location_name][data['user_name']][_type][data['id']]["damage_type"] = damage_type if not result[location_name][data['user_name']][_type][data['id']].get("element", None): result[location_name][data['user_name']][_type][data['id']]["element"] = element_type if added_status_codes != []: if not result[location_name][data['user_name']][_type][data['id']].get("add_status", None): result[location_name][data['user_name']][_type][data['id']]["add_status"] = added_status_codes if int(config['version']) > SETTINGS.exclude_stats_by_version: if damage > 0: if not result[location_name][data['user_name']][_type][data['id']].get("damage", None): result[location_name][data['user_name']][_type][data['id']]["damage"] = {'min': damage, 'max': damage} if damage > result[location_name][data['user_name']][_type][data['id']]["damage"]["max"]: result[location_name][data['user_name']][_type][data['id']]["damage"]["max"] = damage if damage < result[location_name][data['user_name']][_type][data['id']]["damage"]["min"]: result[location_name][data['user_name']][_type][data['id']]["damage"]["min"] = damage elif _type == "status": result[location_name][data['user_name']][_type][data['id']]["type_id"] = data['type'] result[location_name][data['user_name']][_type][data['id']]["name"] = getName(data['id'], data['name']) result[location_name][data['user_name']][_type][data['id']]["icon"] = getStatusIcon(data['id']) if result[location_name][data['user_name']][_type][data['id']].get("duration", None): if int(round(float(data['duration']))) not in result[location_name][data['user_name']][_type][data['id']]["duration"]: result[location_name][data['user_name']][_type][data['id']]["duration"].append(int(round(float(data['duration'])))) result[location_name][data['user_name']][_type][data['id']]["duration"] = sorted(result[location_name][data['user_name']][_type][data['id']]["duration"]) try: result[location_name][data['user_name']][_type][data['id']]["duration"].remove(0) except Exception: pass else: result[location_name][data['user_name']][_type][data['id']]["duration"] = [int(round(float(data['duration'])))] result[location_name][data['user_name']][_type][data['id']]["duration"] = sorted(result[location_name][data['user_name']][_type][data['id']]["duration"]) try: result[location_name][data['user_name']][_type][data['id']]["duration"].remove(0) except Exception: pass elif _type == "headmarker": result[location_name][data['user_name']][_type][data['id']]["target"] = fix_names(data['target']) elif _type == "tether": result[location_name][data['user_name']][_type][data['id']]["from"] = fix_names(data['from']) result[location_name][data['user_name']][_type][data['id']]["to"] = fix_names(data['to']) return result def fix_names(name: str) -> str: return name.replace("[P]", "") #TODO check if username is correct for fights like r4s def check_for_skill_status(known_enemy_names, jline, line, data, config: dict[str, Any], result, error_on_file, file, _type): global g_ignore_status global excluded_enemies global ignore_skill # check if status and/or skill is no in the ignore list or if type is tether/headmarker damage_type = None element_type = None added_status_codes = [] damage = 0 data['id'] = data['id'].upper() if data['user_name'] == "" and (_type == "skill" and data['id'] in ignore_skills_on_empty_enemies): pass elif (_type == "status" and data['id'] not in g_ignore_status) or (_type == "skill" and data['id'] not in ignore_skill) or _type in ["tether", "headmarker"]: if config['location_name']: result = setupDicct(result, config['location_name']) result = setZoneIfPossible(result, config) # if (data['user_name'].lower() not in excluded_enemies and not data['user_name'].lower().endswith(" an deiner seite")) or data['user_name'] == "": if not (config['location_name'].startswith("Himmelssäule") and data['user_name'] in ["Garuda", "Ifrit", "Titan", "Odin"]): # Change data['user_name'] language bnpc__name = getBNPCnName(data['user_name']) data['user_name'] = bnpc__name if "[a]" in data['user_name'].lower(): data['user_name'] = fixSquareBracketOnName(data['user_name'], result[config['location_name']], "") if not result[config['location_name']].get(data['user_name'], None): result[config['location_name']][data['user_name']] = {} if not result[config['location_name']][data['user_name']].get(_type, None): result[config['location_name']][data['user_name']][_type] = {} if data["type"] in ["21", "22"]: _enemy_status = result[config['location_name']][data['user_name']].get('status', {}) damage_type, element_type, damage, added_status_codes = get_attack_type(line, _enemy_status) print_color_yellow(damage_type, element_type, damage) result = addEntryToResults(result, config, data, _type, damage_type, element_type, damage, data["getName"], added_status_codes) elif (_type == "status" and data['id'] in g_ignore_status) or (_type == "skill" and data['id'] in ignore_skill): pass else: print(f"Error for {data['user_name']} using {data['name']}({data['id']}) in {config['location_name']}") error_on_file[file] = "" return result, error_on_file #this will load the normal logdata file to be able to make manual changes easily @timing def get_log_data_if_already_available() -> dict[Any, Any]: try: data = get_any_Logdata(minified=False) return data #return convertLogdataFromClassToJson(getLogdataAsClassNew(data)) except IOError: traceback.print_exc() return {} def cleanup_chocobo_data(result: dict[Any, Any]) -> dict[Any, Any]: for location_name, location_data in result.items(): delete_array = [] for enemy_name, enemiy_skill_types in location_data.items(): #for enemy_name, enemiy_skill_types in location_data.get("enemies", {}).items(): # check for dict to exclude zone element if not isinstance(enemiy_skill_types, dict): continue for enemy_skill_name, enemiy_skills in enemiy_skill_types.items(): if not isinstance(enemiy_skills, dict): continue if enemy_skill_name == "text": continue for _, skill_name in enemiy_skills.items(): if skill_name.get("name", "").startswith("Chocobo-"): delete_array.append(enemy_name) for x in delete_array: if result.get(location_name, {}).get(x, None): del result[location_name][x] return result def cleanup_empty_data(result: dict[Any, Any]) -> dict[Any, Any]: delete_array: list[str] = [] for location_name, enemies in result.items(): if enemies == {}: delete_array.append(location_name) for x in delete_array: del result[x] return result def get_blu_content_name(config: dict[str, str | bool | None | int], result, line, code): x = re.search("„(.+)“ hat begonnen", line) if x and code in ['0839', '0039']: config['location_name'] = x.group(1) if result.get(config['location_name'], None) is None: result[config['location_name']] = {} config['is_bluemage'] = False print(f"\nRenamed BLU instancce from '{config['location_name']}' to '{config['location_name']}'!") return config, result def checkBNPCnames(jline, config: dict[str, str | bool | None | int], result, _type, known_enemy_names): location_name = config['location_name'] try: if jline["lineid"] == "03" and jline["npcnameid"] != "0" and config['placename_locked'] is not None and (str(jline['objectid']).startswith("4") or str(jline['objectid']).startswith("E")): result = setupDicct(result, location_name) result = setZoneIfPossible(result, config) # if not result[location_name].get(_type, None): result[location_name][_type] = {} # isEnemyNotInExcludedList = jline["objectname"].lower() not in [x.lower() for x in excluded_enemies] # try catch will set numberisinrange and enemy_id depending what id is used try: numberIsInRange = 0 < int(jline["npcnameid"]) < 99999 enemy_id = jline["npcnameid"] numberIsInRange = True if numberIsInRange else 0 < int(jline["bnpbasecid"]) < 99999 enemy_id = jline["npcnameid"] if numberIsInRange else jline["bnpbasecid"] except Exception: numberIsInRange = 0 < int(jline["bnpbasecid"]) < 99999 enemy_id = jline["bnpbasecid"] foundBNPCnameAndIDPari = compareBNPCnNameAndID(jline["objectname"], enemy_id) # if not result[location_name][_type].get(enemy_id, None) and isEnemyNotInExcludedList and numberIsInRange and foundBNPCnameAndIDPari: new_name = bnpcname[f"{jline['npcnameid']}"][f'Singular_{compare_lang}'] new_name = fixSquareBracketOnName(new_name, result[location_name], jline['npcnameid'], jline['objectname']) result[location_name][_type][enemy_id] = new_name print(f"'{jline['objectname']}' has ID: {enemy_id}") except Exception as e: traceback.print_tb(e.__traceback__) print(f"'{e}' - {location_name}") return result def check_if_name_already_exists(_id, result_list): for name, data in result_list.items(): # avoid music, zone etc if not isinstance(data, dict): continue # try get id per enemy tmp_id = data.get('id', None) if tmp_id is None or tmp_id == "": return None if isinstance(tmp_id, dict): for i in tmp_id: if i == _id: return name elif isinstance(tmp_id, str): if _id == tmp_id: return name return None DIFFERENT_PRONOUNS = {'0': 'Der', '1': 'Die', '2': 'Das'} DIFFERENT_PRONOUNSS = {'0': 'er', '1': 'e', '2': 'es'} def make_name_readable(entry: str, x: dict[str, str]) -> str: if not x.get("Pronoun", None): return entry name = entry if name.startswith("[t]"): name = name.replace("[t]", DIFFERENT_PRONOUNS[x["Pronoun"]]) elif "[t]" in name: name = name.replace("[t]", DIFFERENT_PRONOUNS[x["Pronoun"]].lower()) if name.startswith("[a]"): name = name.replace("[a]", DIFFERENT_PRONOUNSS[x["Pronoun"]]) elif "[a]" in name: name = name.replace("[a]", DIFFERENT_PRONOUNSS[x["Pronoun"]].lower()) return name def fixSquareBracketOnName(new_name, result_list, _id, line_name=""): global compare_lang global lang # return early if we can get the name fron the line directly #if compare_lang == lang: # if not new_name == "": # return new_name # try to find the correct name # check if we can get id from existing file: if _id == "": try: _id = result_list[new_name]['id'] except Exception: pass # check if we can get id from data files: if _id == "": for key, value in bnpcname.items(): if new_name.lower() == value[f'Singular_{compare_lang}']: _id = key # when we have an id to work with if not _id == "": found_existing_name = check_if_name_already_exists(_id, result_list) if found_existing_name: return found_existing_name # if not available, clear square brackets x_name = bnpcname[_id] if "[a]" in new_name or "[t]" in new_name: new_name = make_name_readable(new_name, x_name) new_name = new_name.replace("[p]", "").replace("[P]", "") return new_name # when we do not have an id and put in a bit of magic for name, data in result_list.items(): a_elements = ["er", "es", "en", "e"] if "[a]" in new_name.lower() else [""] t_elements = ["der", "die", "das"] if "[t]" in new_name.lower() else [""] for rep_a in a_elements: for rep_t in t_elements: t_new_name = new_name.lower().replace("[a]", rep_a) t_new_name = t_new_name.lower().replace("[t]", rep_t) if name.lower() == t_new_name: return name print_color_red(f"Please fix: {new_name} manually ({_id})") return new_name @timing def addAllMissingZones(data): compare_data = [x.lower() for x in data] for _id, content in cfc.items(): if content[f'Name_{lang}']: if content[f'Name_{lang}'].lower() not in compare_data: name = content[f'Name_{lang}'].strip().replace("(Episch)", "(episch)").replace("(Fatal)", "(fatal)") #data[name] = {"": {"skill": {}, "status": {}}} data[name] = {} print(f"Added Zone name for: {name}") return data def add_enemy_id_hp_to_enemy(known_enemy_names, config: dict[str, str | bool | None | int], jline, result): location_name = config['location_name'] # if enemy id not in excludelist npcnameid = patch_name_ids(jline['npcnameid']) if not (int(npcnameid) > 0 and npcnameid not in ["6566", "8228", "8229", "8230", "8313", "5239", "4130", "6146", "6146", "4392", "1402", "6565", "5478", "1399", "8227", "1401", "1398", "1399", "1400", "1401", "1402", "1403", "1404", "5478", "6565", "8227"]): return result # return if pets or new trust members are found if jline['objectname'].lower() in excluded_enemies or jline['objectname'].lower().endswith(" an deiner seite"): return result # create npc name new_name = bnpcname[f"{npcnameid}"]['Singular_de'] enemy_name = checkSpecialCaseForNames(jline['objectid'], known_enemy_names, new_name) enemy_name = fixSquareBracketOnName(enemy_name, result[location_name], npcnameid, jline['objectname']) enemy_name = fixBNPCName(enemy_name) # if enemy not in location if not result[location_name].get(enemy_name, None): result[location_name][enemy_name] = {} if not result[location_name].get("combatants", None): result[location_name]["combatants"] = {} result[location_name]["combatants"][npcnameid] = enemy_name # handle id stuff _id = result[location_name][enemy_name].get('id', "") # this assignment is needed in case i need to overwrite it manually on patch if _id == "": result[location_name][enemy_name]['id'] = npcnameid elif isinstance(_id, list) and npcnameid not in _id: _id.append(npcnameid) _id = sorted(_id) result[location_name][enemy_name]['id'] = _id elif isinstance(_id, str) and _id and not _id == npcnameid: _id = [_id, npcnameid] _id = sorted(_id) result[location_name][enemy_name]['id'] = _id # skip hp if version is to low due to state squish in 6.0 if int(config['version']) > SETTINGS.exclude_stats_by_version: # check if enemy has maxhp attribute if not result[location_name][enemy_name].get('maxHP', None): result[location_name][enemy_name]['maxHP'] = int(jline['maxhp']) if not result[location_name][enemy_name].get('minHP', None) and int(jline['maxhp']) > 99: result[location_name][enemy_name]['minHP'] = int(jline['maxhp']) # set minhp value if new value is lower than original value if int(jline['maxhp']) <= result[location_name][enemy_name].get('minHP', int(jline['maxhp'])) and int(jline['maxhp']) > 99: result[location_name][enemy_name]['minHP'] = int(jline['maxhp']) # set maxhp value if new value is higher than original value if int(jline['maxhp']) > result[location_name][enemy_name]['maxHP']: result[location_name][enemy_name]['maxHP'] = int(jline['maxhp']) return result def handleBaldesionArsenalStuff(line, config: dict[str, str | bool | None | int]): # get location for Castrum Lacus Litore if "Eureka Hydatos" == config['location_name'] and "Du erleidest den Effekt von  Wiederbelebungsbegrenzung." in line: print("Found Eureka Baldesions Arsenal") config['is_baldesion_active'] = True config['saved_location_when_entering_special_zone'] = config['location_name'] config['location_name'] = "Eureka Baldesions Arsenal" elif config['is_baldesion_active'] and ("Du erholst dich von dem Effekt von  Wiederbelebungsbegrenzung" in line or "hat Proto-Yadis besiegt" in line or "hast Proto-Yadis besiegt" in line): print("Exit Eureka Baldesions Arsenal") config['is_baldesion_active'] = False config['location_name'] = config['saved_location_when_entering_special_zone'] return config def handleBozjaStuff(line: str, config: dict[str, str | bool | None | int]) -> dict[str, str | bool | None | int]: # get location for Castrum Lacus Litore if "Bozja-Südfront" == config['location_name'] and "Du nimmst an „Sturm auf Castrum Lacus Litore“ teil." in line: print("Found Castrum Lacus Litore") config['is_castrum_active'] = True config['saved_location_when_entering_special_zone'] = config['location_name'] config['location_name'] = "Castrum Lacus Litore" elif config['is_castrum_active'] and ("hat Dawon besiegt" in line or "hast Dawon besiegt" in line): print("Exit Castrum Lacus Litore") config['is_castrum_active'] = False config['location_name'] = config['saved_location_when_entering_special_zone'] return config def handleZladnorStuff(line: str, config: dict[str, str | bool | None | int]) -> dict[str, str | bool | None | int]: # get location for Castrum Lacus Litore if "Zadnor-Hochebene" == config['location_name'] and "Du nimmst an „Sturm auf die Dalriada“ teil." in line: print("Found Dalriada") config['is_dalriada_active'] = True config['saved_location_when_entering_special_zone'] = config['location_name'] config['location_name'] = "Die Dalriada" elif config['is_dalriada_active'] and ("hat das Diablo-Armament besiegt" in line or "hast das Diablo-Armament besiegt" in line): print("Exit Dalriada") config['is_dalriada_active'] = False config['location_name'] = config['saved_location_when_entering_special_zone'] return config def handleKreszentiaTurmDesBlutesStuff(line, config: dict[str, str | bool | None | int]): # get location for Castrum Lacus Litore if "Das südliche Kreszentia" == config['location_name'] and 'Du nimmst an der kritischen Begegnung „Der Turm des Blutes“ teil.' in line: print("Found Turm des Blutes") config['is_kreszentiaturmdesblutes_active'] = True config['saved_location_when_entering_special_zone'] = config['location_name'] config['location_name'] = "Turm des Blutes" elif config['is_kreszentiaturmdesblutes_active'] and ("Du erholst dich von dem Effekt von  Fester Job" in line or "hat Magitaurus besiegt" in line or "hast Magitaurus besiegt" in line): print("Exit Turm des Blutes") config['is_kreszentiaturmdesblutes_active'] = False config['location_name'] = config['saved_location_when_entering_special_zone'] return config def check_language_stuff(line: str, config: dict[str, str | bool | None | int]) -> dict[str, str | bool | None | int]: tmp_lang: str = getLanguageToImport(line) if f"Language ID: {tmp_lang}" in line: config['found_correct_language'] = True #print_color_green(f"Found Language Code {tmp_lang}") elif "Language ID: " in line: config['found_correct_language'] = False print_color_red(f"Incorrect Languagecode found, change settings to {tmp_lang}") print_color_red(line) #compare_lang = "de" return config def handle_example_lines(line_dict: dict[str, str], logname: str, line: str) -> None: global example_lines # seperate 00 into an own array if line_dict["lineid"] == "00": if not example_lines.get("0" + line_dict["lineid"] + " - " + logname, None): example_lines["0" + line_dict["lineid"] + " - " + logname] = {} if not example_lines["0" + line_dict["lineid"] + " - " + logname].get(line_dict["code"], None): example_lines["0" + line_dict["lineid"] + " - " + logname][line_dict["code"]] = line_dict elif len(line_dict["lineid"]) > 1: key_string = line_dict["lineid"] + " - " + logname if len(line_dict["lineid"]) < 3: key_string = "0" + line_dict["lineid"] + " - " + logname if not example_lines.get(key_string, None): example_lines[key_string] = line_dict else: print_color_red(line) def split_line_to_json(line: str, file_version: int) -> dict[str, str]: line_dict, logname = splitLogLineByVersion(file_version, line) # type: ignore handle_example_lines(line_dict, logname, line) # type: ignore try: del line_dict['timestamp'] except Exception: line_dict.__deleteitems__('timestamp') return line_dict def getPlacenameLocked(config: dict[str, str | bool | None | int], line: str, code: str) -> dict[str, str | bool | None | int]: # look for close message in both german and english if code == "0839": m = re.search("Noch 15 Sekunden, bis sich (?:(?:der|die|das) )?(?:Zugang zu(?:[rm]| den)? )?(.*) schließt", line) n = re.search("(.*) will be sealed off(?: in (?:[0-9]+ seconds)?)?", line) if m: config['placename_locked'] = m.group(1) if n: config['placename_locked'] = getPlacenameName(n.group(1)) # if n or m: # print(F"placename_locked: {config['placename_locked']} - {jline['line']}") return config def addClassJobStatus(line, jline, result, user_to_classes, config: dict[str, str | bool | None | int]): global filter_player_status global status # ignore stuff if in a special zone if config['is_baldesion_active'] or config['is_castrum_active'] or config['is_dalriada_active'] or config['is_kreszentiaturmdesblutes_active'] or \ config['location_name'] in ["Zadnor-Hochebene", "Bozja-Südfront"] or \ "Eureka" in config['location_name'] or config['location_name'] == "": return result _, _, _, added_status_codes = get_attack_type(line, {}) if not result["Klassen"].get(user_to_classes[jline['sourcename']], None): result["Klassen"][user_to_classes[jline['sourcename']]] = {"skill": {}, "status": {}} for key in added_status_codes: if key in g_ignore_status: continue if key not in result["Klassen"][user_to_classes[jline['sourcename']]]["status"]: tmp = { "name": status[str(int(key, 16))]["Name_de"], "type_id": jline["lineid"], "duration": None, # this could be updated on status line if debug was already available "icon": status[str(int(key, 16))]["Icon"], "was_added_by_action": jline["actionid"] } result["Klassen"][user_to_classes[jline['sourcename']]]["status"][key.upper()] = tmp #result["Klassen"][user_to_classes[jline['sourcename']]]["status"][jline["statusid"].upper()] = tmp return result def OLDaddClassJobStatus(jline, result, user_to_classes, config: dict[str, str | bool | None | int]): global filter_player_status global status # add user based status if id starts not with e or 4 if config['is_baldesion_active'] or config['is_castrum_active'] or config['is_dalriada_active'] or config['is_kreszentiaturmdesblutes_active'] or \ config['location_name'] in ["Zadnor-Hochebene", "Bozja-Südfront"] or \ "Eureka" in config['location_name'] or config['location_name'] == "": # print(config) pass elif jline['sourcename'] and jline["statusid"] not in filter_player_status and jline["statusid"] not in g_ignore_status: if jline["statusid"].upper() in result["Klassen"][user_to_classes[jline['sourcename']]]["status"]: #return if skill was not added by action line 21 if not result["Klassen"][user_to_classes[jline['sourcename']]]["status"].get(jline["statusid"].upper(), None): return result result["Klassen"][user_to_classes[jline['sourcename']]]["status"][jline["statusid"].upper()]['duration'] = [int(jline["duration"].split(".")[0])] return result #TODO Check here for mostranslations enemy_line_itterator = [ ("npcyell_ids", npcyell), ("instancecontenttextdata_ids", instancecontenttextdata), ] def getEnemyChatLogLines(jline: dict[str, str], location: str, lresult: dict[str, Any]) -> dict[str, Any]: global enemy_line_itterator if "\ue091" in jline['name']: return lresult # code lsit below are all chats that are used by player + the echo chat if jline['code'] in ["000A", "001E", "000B", "000C", "000D", "000E", "000F", "0018", "0024", "0025", "0065", "0066", "0067", "0068", "0069", "006A", "006B", "0010", "0011", "0012", "0013", "0014", "0015", "0016", "0017", "001B", "001D", "001C" ,'0038', '0001']: return lresult if jline['code'] in ["003D"] and (jline["name"] in ['Kiritokara', 'Vicktoriachan', 'Keichan', 'Akurosiahulk', "K'urumichan", 'Chloechan', 'Soreychan', 'Leilahchan', 'Sokaauri', 'Lilymausi'] or jline["name"].startswith("Transporter")): return lresult if jline["name"]: #if not lresult[location].get(jline['name'].title(), None): # return lresult # filter out all non bnpc elements e.g. player if jline['name'] not in bnpc_names: return lresult # remove npcs an deiner seite if " an deiner seite" in jline['name'].lower(): return lresult enemy_name: str = jline["name"].title() textx: str = jline['line'].strip() if not lresult[location].get(enemy_name, None): lresult[location][enemy_name] = {} if not lresult[location][enemy_name].get('text', None): lresult[location][enemy_name]['text'] = { 'npcyell_ids': [], 'instancecontenttextdata_ids': [], 'fateevent_ids': [], 'old': [] } for ele in enemy_line_itterator: fieldname, array = ele for key, value in array.items(): de_val: str = value['Text_de'].replace("\n\n", " ").strip() if textx == de_val: if key not in lresult[location][enemy_name]['text'][fieldname]: lresult[location][enemy_name]['text'][fieldname].append(key) print_color_green(f"[getEnemyChatLogLines] {enemy_name}: {jline['line']}") return lresult if not lresult[location][enemy_name]['text'].get('fateevent_ids', None): lresult[location][enemy_name]['text']['fateevent_ids'] = [] for key, value in fateevent.items(): for i, field in enumerate(value['Text_de']): de_val: str = field.replace("\n\n", " ").strip() if textx == de_val: if key not in lresult[location][enemy_name]['text']['fateevent_ids']: lresult[location][enemy_name]['text']['fateevent_ids'].append(key) print_color_green(f"[getEnemyChatLogLines] {enemy_name}: {jline['line']}") return lresult if jline['line'] not in lresult[location][enemy_name]['text']['old']: lresult[location][enemy_name]['text']['old'].append(jline['line']) print_color_red(f"[getEnemyChatLogLines] {jline['line']}") #if jline['line'] not in lresult[location][enemy_name]['text']: # lresult[location][enemy_name]['text'].append(jline['line']) return lresult def getZoneMusic(jline, location, result): valid_line = None if jline["line"].startswith("Orchestrion: Now playing "): valid_line = jline["line"].replace("Orchestrion: Now playing ", "")[:-1] if jline["line"].startswith("[Orchestrion] Aktuell läuft: "): valid_line = jline["line"].replace("[Orchestrion] Aktuell läuft: ", "")[:-1] if valid_line: name = valid_line if name == "None" or name == "???": return result if not result[location].get('music', None): result[location]['music'] = [] if name not in result[location]['music']: result[location]['music'].append(name) result[location]['music'] = sorted(result[location]['music']) return result def getVersionFromLogfile(line): if "|FFXIV PLUGIN VERSION:" in line: version = line.split("|FFXIV PLUGIN VERSION: ")[1] version = version.split("|")[0] version = version.split(", CLIENT MODE")[0] else: version = line.split("|FFXIV_ACT_Plugin Version: ")[1] version = version.split(" (")[0] version = version.split(".") version.insert(2, "0") version = "".join(version) version = int(version) #print_color_green(f"Found version: {version}") return version def setDefaultConfig() -> dict[str, Any]: tmp: dict[str, Any] = { 'version': 0, 'is_bluemage': False, 'is_baldesion_active': False, 'is_castrum_active': False, 'is_dalriada_active': False, 'is_kreszentiaturmdesblutes_active': False, 'saved_location_when_entering_special_zone': "", 'found_correct_language': False, 'switchedLanguageCode': False, 'placename_locked': None, 'location': "", 'location_id': "", 'location_name': "", "player": "", "current_map": "", "old_map": "", "current_time_of_mapchange": "", "old_time_of_mapchange": "", "fates_to_compare": [], } return tmp def patch_name_ids(npcnameid) -> str: return npcnameid def fix_common_issue_name(jline: dict, known_enemy_names: dict) -> dict: global compare_lang # this assignment is needed in case i need to overwrite it manually on patch npcnameid = patch_name_ids(jline['npcnameid']) value = bnpcname[f"{npcnameid}"][f'Singular_{compare_lang}'] # hespheros, hephaistos, Tosender Donner P1, Heulende Klinge, Lindwurm if npcnameid in ["10742", "11399", "13057", "13843", "14378"]: value = value + " I" # hespheros, hephaistos, Tosender Donner P2, Heulende Klinge elif npcnameid in ["10744", "11402", "13058", "13844", "14379"]: value = value + " II" # omega elif npcnameid in ["XXXXXXXXXX"]: value = value + " III" known_enemy_names[jline['objectid']] = value return known_enemy_names #TODO check here for mistranslation def checkSpecialCaseForNames(jline_id: str, known_enemy_names: dict, npc_name: str) -> str: if known_enemy_names.get(jline_id, None): npc_name: str = known_enemy_names[jline_id] if npc_name == "": npc_name return npc_name def AddCombatant(result, user_to_classes, jline, known_enemy_names, config): # Skip Trainingspuppe if jline['npcnameid'] in ["541"]: return # when ownerid is set this usually means chocobo, minion or mount, which we do not care about elif jline["ownerid"].startswith("10"): return # when user is a player elif jline["objectid"][0] == "1": user_to_classes[jline['objectname']] = cj[str(int(jline['jobid'], 16))][f"Name_{compare_lang}"] # when user is a npc else: known_enemy_names = fix_common_issue_name(jline, known_enemy_names) result = add_enemy_id_hp_to_enemy(known_enemy_names, config, jline, result) result = checkBNPCnames(jline, config, result, "combatants", known_enemy_names) def workOnLogLines(file, line:str, jline: dict[str, Any], result: dict[str, Any], config: dict[str, Any], user_to_classes, error_on_file, known_enemy_names): # ChatLog if jline["lineid"] == "00": # handleenemy charlines result = getEnemyChatLogLines(jline, config['location_name'], result) result = getZoneMusic(jline, config['location_name'], result) # ChangePrimaryPlayer elif jline["lineid"] == "02": config['player'] = jline['playername'] # AddCombatant - check case for line entry 03 (line.split("|")[9] will be the key from bnpcid) elif jline["lineid"] == "03": AddCombatant(result, user_to_classes, jline, known_enemy_names, config) # PlayerStats - e.g. jobswitch elif jline["lineid"] == "12": user_to_classes[config['player']] = cj[jline['jobid']][f"Name_{compare_lang}"] # StartsCasting, ActionEffect, AOEActionEffect, CancelAction elif jline["lineid"] in ["20", "21", "22", "23"]: # skip if source id is a player if jline['sourceid'][0] not in ["E", "4"]: if jline["lineid"] in ["21", "22"] and args.include_buffs: result = addClassJobStatus(line, jline, result, user_to_classes, config) return file, jline, result, config, user_to_classes, error_on_file, known_enemy_names data = {'type': jline['lineid'], 'user_name': jline['sourcename'].title(), 'name': jline['actionname'], 'id': jline['actionid'], 'getName': getActionName} data['user_name'] = checkSpecialCaseForNames(jline['sourceid'], known_enemy_names, data['user_name']) result, error_on_file = check_for_skill_status(known_enemy_names, jline, line, data, config, result, error_on_file, file, "skill") # StatusAdd, StatusRemove elif jline["lineid"] in ["26", "30"]: # skip if source id is a player if jline['sourceid'][0] not in ["E", "4"]: # add status if status is applied only #if jline['lineid'] == "26" and args.include_buffs: # result = OLDaddClassJobStatus(jline, result, user_to_classes, config) pass else: data = {'type': jline['lineid'], 'user_name': jline['sourcename'].title(), 'name': jline['statusname'], 'id': jline['statusid'], 'duration': jline['duration'], 'getName': getStatusName} data['user_name'] = checkSpecialCaseForNames(jline['sourceid'], known_enemy_names, data['user_name']) result, error_on_file = check_for_skill_status(known_enemy_names, jline, line, data, config, result, error_on_file, file, "status") # Network6D - e.g zonereset on wipe elif jline["lineid"] == "33": if not result[config['location_name']].get('contentzoneid', None): result[config['location_name']]['contentzoneid'] = [] if jline['contentzoneid'] not in result[config['location_name']]['contentzoneid']: result[config['location_name']]['contentzoneid'].append(jline['contentzoneid']) # clear known enemy names if run is done or wipe if jline['command'] in ["40000002", "40000003", "4000000F", "40000010", "40000011", "40000005"]: known_enemy_names = {} # HeadMarker elif jline["lineid"] == "27": target = "Player" if jline['targetid'][0] == "1" else "NPC" data = {'type': jline['lineid'], 'user_name': "", 'target': target, 'id': jline['id'], 'getName': empty} result, error_on_file = check_for_skill_status(known_enemy_names, jline, line, data, config, result, error_on_file, file, "headmarker") # Tether elif jline["lineid"] == "35": _from: str = "Player" if jline['sourceid'][0] == "1" else getBNPCnName(jline['sourcename']) ___to: str = "Player" if jline['targetid'][0] == "1" else getBNPCnName(jline['targetname']) data = {'type': jline['lineid'], 'user_name': "", 'from': _from, 'to': ___to, 'id': jline['id'], 'getName': empty} result, error_on_file = check_for_skill_status(known_enemy_names, jline, line, data, config, result, error_on_file, file, "tether") # MapChange elif jline["lineid"] == "01": splitline: list[str] = line.split("|") config['current_map'] = config['location_name'] config['current_time_of_mapchange'] = dp.parse(splitline[1]) #config['current_time_of_mapchange'] = datetime.fromisoformat(splitline[1]) for f in config['fates_to_compare']: _time, _type, fatename = f diff: timedelta = config['current_time_of_mapchange'] - _time diff2: timedelta = _time - config['old_time_of_mapchange'] if diff.total_seconds() > 5 and diff2.total_seconds() < 5: if not result[config['old_map']].get("fates", None): result[config['old_map']]['fates'] = [] if fatename not in result[config['old_map']]['fates']: result[config['old_map']]['fates'].append(fatename) config['fates_to_compare'] = [] # set current values to old values config['old_map'] = config['current_map'] config['old_time_of_mapchange'] = config['current_time_of_mapchange'] # Fates elif jline["lineid"] == "258": splitline: list[str] = line.split("|") fate: str = str(int(jline['fateId'], 16)) current_time: datetime = dp.parse(splitline[1]) config['fates_to_compare'].append((current_time, jline["category"], fate)) return file, jline, result, config, user_to_classes, error_on_file, known_enemy_names def doConfigurationStuffAndLookups(line:str, jline: list[str], result, config: dict[str, str | bool | None | int], user_to_classes, error_on_file): if line == "": return line, jline, result, config, user_to_classes, error_on_file if line.startswith('253|'): config['version'] = getVersionFromLogfile(line) if config['version'] == 0: return line, jline, result, config, user_to_classes, error_on_file jline = split_line_to_json(line, config['version']) # Line when ACT was restarted if jline["lineid"] == "253": config = check_for_act_restart(jline["line"], config) user_to_classes = {} # if language code was not correct if jline["lineid"] == "249": config = check_language_stuff(jline["line"], config) # if not (config['found_correct_language'] or config['switchedLanguageCode']): if config['found_correct_language'] is False: return line, jline, result, config, user_to_classes, error_on_file # Line for when instance is changed if jline["lineid"] == "01": config, result = check_for_location(jline, config, result) user_to_classes = {} # get location for blu if config["is_bluemage"] and jline["lineid"] == "00": config, result = get_blu_content_name(config, result, jline['line'], jline['code']) if jline["lineid"] == "00": # get specials for eureka und bozja config = handleBaldesionArsenalStuff(jline['line'], config) config = handleBozjaStuff(jline['line'], config) config = handleZladnorStuff(jline['line'], config) config = handleZladnorStuff(jline['line'], config) config = handleKreszentiaTurmDesBlutesStuff(jline['line'], config) # get placename from closing message config = getPlacenameLocked(config, jline['line'], jline['code']) return line, jline, result, config, user_to_classes, error_on_file def get_new_log_data_to_json(args): global compare_lang global result print_color_blue("[GNLDTJ] Get Logdata if already available") result = get_log_data_if_already_available() print_color_blue("[GNLDTJ] Add all missing zones") result = addAllMissingZones(result) error_on_file = {} files = glob(f"{args.main_path}/{args.sub_path}/{args.file_regex}") number_of_files = len(files) counter_of_files = 0 # print(files) files_with_error = [] for file in files: try: counter_of_files += 1 if args.skip_files >= counter_of_files: print_color_red(f"Skip file {counter_of_files}/{number_of_files}: {file}") continue print_color_green(f"Work on file {counter_of_files}/{number_of_files}: {file}") config: dict[str, str | bool | None | int] = setDefaultConfig() user_to_classes = {} jline = None known_enemy_names = {} for line in tqdm(iterable=open(file, encoding="utf8"), total=get_num_lines(file)): try: line, jline, result, config, user_to_classes, error_on_file = doConfigurationStuffAndLookups(line, jline, result, config, user_to_classes, error_on_file) # only if a propper location is found, start working on the loglines itself to gather the data if not config['found_correct_language']: continue if config['location'] and not config['is_bluemage']: file, jline, result, config, user_to_classes, error_on_file, known_enemy_names = workOnLogLines(file, line, jline, result, config, user_to_classes, error_on_file, known_enemy_names) except KeyError as exception: print_color_red(f"{line=} => {exception=}") print_color_red(f"{jline=} => {exception=}") traceback.print_exc() except UnboundLocalError as exception: print_color_red(f"{line=} => {exception=}") print_color_red(f"{jline=} => {exception=}") except Exception: print("Exception in user code:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("*" * 60) print(line) print("*" * 60) print("-" * 60) # write logdata files at least every 10 files to loose data in case of a crash if counter_of_files % 10 == 0: writeLogdataFile(result, lang, files_with_error, args) result = cleanup_chocobo_data(result) result = cleanup_enemy_data(result) writeLogdataFile(result, lang, files_with_error, args) def print_custom_test(data): results = {} #print(result['Westliches Thanalan']['fates']) print("This print originates from function [print_custom_test] and is only printed when skip_file_write is true") for enemy, value in data['Westliches Thanalan'].items(): #for enemy, value in data['Gymnasion Agonon'].items(): results[enemy] = value try: if results[enemy].get("skill", None): del results[enemy]['skill'] if results[enemy].get("status", None): del results[enemy]['status'] if results[enemy].get("headmarker", None): del results[enemy]['headmarker'] if results[enemy].get("tether", None): del results[enemy]['tether'] if results[enemy].get("text", None): del results[enemy]['text'] if results[enemy].get("id", None): del results[enemy]['id'] if results[enemy].get("maxHP", None): del results[enemy]['maxHP'] if results[enemy].get("minHP", None): del results[enemy]['minHP'] except: pass print_pretty_json(results) def cleanup_enemy_data(result: dict[str, Any]) -> dict[str, Any]: for zone_name, zone_data in result.items(): #if zone_name != "Seenland": # continue enemies_to_remove: list[str] = [] for name, enemy_data in zone_data.items(): if name in ['contentzoneid', 'fates', 'music', 'combatants', 'zone']: continue if enemy_data.get('id', None): continue #if not enemy_data.get('skill', None) and not enemy_data.get('status', None) and enemy_data.get("text", None): # enemies_to_remove.append(name) only_attack: list[str] = [k for k,_ in enemy_data.get('skill', {}).items()] if len(only_attack) == 1 and only_attack[0] == "367": enemies_to_remove.append(name) for enemy in enemies_to_remove: try: print(f"Remove: {enemy}") del result[zone_name][enemy] except Exception as e: print(e) remove_combatants: list[str] = [] for k,_ in zone_data.get('combatants', {}).items(): if bnpcname.get(k, None): continue remove_combatants.append(k) for k in remove_combatants: try: del result[zone_name]['combatants'][k] except Exception as e: print(e) return result def writeLogdataFile(lresult: dict[str, Any], llang: str, files_with_error: list[str], largs: Namespace) -> None: if largs.skip_file_write: print_custom_test(lresult) return print_color_green("Write logdata!") print_color_red(files_with_error) nresult: dict[str, Any] = lresult writeJsonFile(f"../json/logdata_{llang}.json", nresult, sort_keys=True, sort_sub_keys=True) #writeJsonFile(f"../json/logdata_{llang}_minified.json", nresult, minified=True) writeJsonFile(f"../json/logdata_{llang}_minified.json", nresult, keys_per_line=True, sort_keys=True, sort_sub_keys=True) writeJsonFile(f"../json/logdata_{llang}_new_minified.json", convertLogdataFromClassToJson(getLogdataAsClassNew(nresult)), keys_per_line=True, sort_keys=True, sort_sub_keys=True) #with open(f"../json/logdata_{lang}.json", "w", encoding="UTF-8") as f: # f.write(result3) def print_examples_sorted(_example_lines: dict[str, Any]) -> None: keys: list[str] = sorted(_example_lines.keys()) tmp_dict: dict[str, Any] = {} for key in keys: # sort elements on 000 as prettyprint > sortkeys changes lowest order if key == "000 - ChatLog ": tmp_dict[key] = {} for x in sorted(_example_lines[key]): tmp_dict[key][x] = _example_lines[key][x].to_json() else: tmp_dict[key] = _example_lines[key] print(tmp_dict) print_pretty_json(tmp_dict, sort_keys=False) def manual_args_overwrite(_args: Namespace) -> Namespace: _args.sub_path = 'All_Log_Files' _args.skip_file_write = True _args.skip_files = '30' #_args.print_log_lines = True return _args @timing def checkArgumentsAndsetDefaults(_args: Namespace) -> None: if not _args.main_path: _args.main_path = "." if not _args.sub_path: _args.sub_path = "Neuer Ordner" if not _args.file_regex: _args.file_regex = "Network_*.log" if not _args.skip_files: _args.skip_files = 0 else: _args.skip_files = int(_args.skip_files) if _args.file_language in ["de", "en", "fr", "ja", "cn", "ko"]: global compare_lang compare_lang = _args.file_language example_lines = {} if __name__ == "__main__": example_usage: str = "" parser = argparse.ArgumentParser( description="Recreates the logdata json file", epilog=example_usage, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("-mp", "--main-path", help="main path to file") parser.add_argument("-sp", "--sub-path", help="sub path to file") parser.add_argument("-fr", "--file-regex", help="file regex") parser.add_argument("-fl", "--file-language", help="file language") parser.add_argument("-ib", "--include-buffs", action='store_true', help="file language") parser.add_argument("-sfw", "--skip-file-write", action='store_true', help="If set to true, logdata will not be saved") parser.add_argument("-sf", "--skip-files", help="Skip already processed files") parser.add_argument("-pll", "--print-log-lines", action='store_true', help="Print log line examples") args: Namespace = parser.parse_args() #args = manual_args_overwrite(args) print_color_blue("Check arguments and set defaults") checkArgumentsAndsetDefaults(args) print_color_blue("Get new Log data to json") get_new_log_data_to_json(args) if args.print_log_lines: print_examples_sorted(example_lines)