package datapreprocessor import ( "errors" "fmt" "io/ioutil" "os" "path/filepath" "strings" "time" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) var dataTypes = []string{"adventure_collectReward", "adventure_endBattle", "adventure_find", "adventure_getActiveData", "adventure_getGlobalBuffs", "adventure_getInfo", "adventure_getPassed", "adventure_start", "adventure_turnCollectBuff", "adventure_turnStartBattle", "arenaAttack", "arenaCheckTargetRange", "arenaFindEnemies", "arenaGetAll", "artifactChestOpen", "artifactGetChestLevel", "battleGetByType", "battleGetReplay", "billingGetAll", "billingGetLast", "bossGetAll", "bossOpenChest", "bossRaid", "campaignStoryGetList", "chatAcceptChallenge", "chatGetAll", "chatGetInfo", "chatGetTalks", "chatSendChallenge", "chatSendText", "chatServerSubscribe", "chatServerUnsubscribe", "chatSetSettings", "chestBuy", "clanGetActivityRewardTable", "clanGetActivityStat", "clanGetInfo", "clanGetPrevData", "clanGetWeeklyStat", "clanWarAttack", "clanWarEndBattle", "clanWarGetAvailableHistory", "clanWarGetBriefInfo", "clanWarGetDayHistory", "clanWarGetDefence", "clanWarGetInfo", "clanWarGetWarlordInfo", "consumableUseHeroXp", "consumableUseLootBox", "consumableUsePetXp", "consumableUseStamina", "dailyBonusFarm", "dailyBonusGetInfo", "dungeonEndBattle", "dungeonGetInfo", "dungeonSaveProgress", "dungeonStartBattle", "expeditionFarm", "expeditionGet", "expeditionSendHeroes", "freebieCheck", "freebieHaveGroup", "friendsGetInfo", "friendsSendDailyGift", "getTime", "grandAttack", "grandCheckTargetRange", "grandFindEnemies", "hallOfFameGetTrophies", "heroArtifactLevelUp", "heroEnchantRune", "heroesMerchantGet", "heroGetAll", "heroSkinUpgrade", "heroTitanGiftLevelUp", "heroUpgradeSkill", "inventoryCraftFragments", "inventoryCraftRecipe", "inventoryExchangeHeroArtifacts", "inventoryExchangeStones", "inventoryExchangeTitanStones", "inventoryGet", "lootBoxBuy", "mailFarm", "mailGetAll", "mechanicAvailability", "missionGetAll", "missionGetReplace", "missionRaid", "newYearGetInfo", "offerFarmReward", "offerGetAll", "offerTrialPay", "pet_chestOpen", "pet_getAll", "pet_getChest", "pet_getPotionDailyBuyCount", "pet_levelUpItem", "pirateTreasureIsAvailable", "playable_getAvailable", "questFarm", "questGetAll", "questGetEvents", "questGetPaymentDependent", "refillableAlchemyUse", "refillableBuyStamina", "registration", "settingsGetAll", "settingsSet", "shopBuy", "shopGet", "shopGetAll", "socialQuestGetInfo", "splitGetAll", "stashClient", "subscriptionFarm", "subscriptionGetInfo", "teamGetAll", "teamGetMaxUpgrade", "titanArenaCheckForgotten", "titanArenaCompleteTier", "titanArenaEndBattle", "titanArenaEndRaid", "titanArenaFarmDailyReward", "titanArenaGetChestReward", "titanArenaGetDailyReward", "titanArenaGetLog", "titanArenaGetStatus", "titanArenaStartBattle", "titanArenaStartRaid", "titanArtifactChestOpen", "titanArtifactGetChest", "titanArtifactLevelUp", "titanArtifactLevelUpStarmoney", "titanGetAll", "titanGetSummoningCircle", "titanSpiritGetAll", "titanUseSummonCircle", "tower_farmPointRewards", "tower_farmSkullReward", "towerGetInfo", "tower_getSkullReward", "towerNextChest", "towerOpenChest", "tutorialGetInfo", "userGetAvailableAvatars", "userGetAvailableStickers", "userGetInfo", "userMergeGetStatus", "zeppelinGiftFarm", "zeppelinGiftGet", } var dataTypesActive = []string{} // NewPreProcessor ... func NewPreProcessor(path string, debug bool) *PreProcessor { p := new(PreProcessor) p.Dir = path path, _ = filepath.Abs(path) p.Debug = debug return p } // PreProcessor ... type PreProcessor struct { Dir string Debug bool } // RunPath ... func (p *PreProcessor) RunPath() { p.main() } // RunBody ... func (p *PreProcessor) RunBody(data []byte) error { return p.processJSONBody(data) } // main to run it all... func (p *PreProcessor) main() { fmt.Printf("START PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999")) err := filepath.Walk(p.Dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !strings.Contains(path, "\\2020-") && !strings.Contains(path, "/2020-") { return nil } if info.IsDir() { return nil } if strings.HasSuffix(info.Name(), ".json") { err = p.processJSONFile(path, info) if err != nil { fmt.Println(err) } } return nil }) if err != nil { fmt.Println(err) } fmt.Printf("END PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999")) } func (p *PreProcessor) processJSONBody(json []byte) error { date := gjson.GetBytes(json, `response.body.date`) if date.Exists() { req := gjson.GetBytes(json, `request`) if !req.Exists() { return errors.New("no request") } reqHeaders := req.Get(`headers`) if !reqHeaders.Exists() { return errors.New("no request.headers") } for _, identSearch := range dataTypes { call := req.Get(`body.calls.#(name=="` + identSearch + `")`) if !call.Exists() { continue } ident := call.Get("ident") if !ident.Exists() { continue } data := gjson.GetBytes(json, `response.body.results.#(ident=="`+ident.String()+`")`) if !data.Exists() { data2 := gjson.GetBytes(json, `response.body`) if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" { return fmt.Errorf("JSON with invalide signature") } continue } saveDir := filepath.Join(p.Dir, "data", identSearch) saveFile := filepath.Join(saveDir, date.Raw+"_"+reqHeaders.Get("X-Auth-User-Id").String()+".json") os.MkdirAll(saveDir, os.ModePerm) if _, err := os.Stat(saveFile); !os.IsNotExist(err) { continue } var sdata []byte var errdata error sdata, errdata = sjson.SetRawBytes(sdata, "date", []byte(date.Raw)) if errdata != nil { return errdata } sdata, errdata = sjson.SetRawBytes(sdata, "request.headers", []byte(reqHeaders.Raw)) if errdata != nil { return errdata } sdata, errdata = sjson.SetRawBytes(sdata, "request.call", []byte(call.Raw)) if errdata != nil { return errdata } sdata, errdata = sjson.SetRawBytes(sdata, "request.ident", []byte(ident.Raw)) if errdata != nil { return errdata } sdata, errdata = sjson.SetRawBytes(sdata, "response", []byte(data.Raw)) if errdata != nil { return errdata } err := ioutil.WriteFile(saveFile, sdata, os.ModePerm) if err != nil { fmt.Printf("%v - %v - %v\n", saveDir, saveFile, err) } } } else { data2 := gjson.GetBytes(json, `response.body`) if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" { return fmt.Errorf("%s: JSON with invalide signature", "bodyData") } } return nil } func (p *PreProcessor) processJSONFile(path string, info os.FileInfo) error { fmt.Printf("START PreProcessing of %s at %s\n", path, time.Now().UTC().Format("2006-01-02 15:04:05.999999")) json, err := p.readJSON(path) if err != nil { return err } return p.processJSONBody(json) } func (p *PreProcessor) readJSON(path string) ([]byte, error) { // read file data, err := ioutil.ReadFile(path) if err != nil { return []byte(""), err } return data, nil }