backend-server/server/services/datapreprocessor/datapreprocessor.go

package datapreprocessor

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/tidwall/gjson"
	"github.com/tidwall/sjson"
)

var dataTypes = []string{"adventure_collectReward",
	"adventure_endBattle",
	"adventure_find",
	"adventure_getActiveData",
	"adventure_getGlobalBuffs",
	"adventure_getInfo",
	"adventure_getPassed",
	"adventure_start",
	"adventure_turnCollectBuff",
	"adventure_turnStartBattle",
	"arenaAttack",
	"arenaCheckTargetRange",
	"arenaFindEnemies",
	"arenaGetAll",
	"artifactChestOpen",
	"artifactGetChestLevel",
	"battleGetByType",
	"battleGetReplay",
	"billingGetAll",
	"billingGetLast",
	"bossGetAll",
	"bossOpenChest",
	"bossRaid",
	"campaignStoryGetList",
	"chatAcceptChallenge",
	"chatGetAll",
	"chatGetInfo",
	"chatGetTalks",
	"chatSendChallenge",
	"chatSendText",
	"chatServerSubscribe",
	"chatServerUnsubscribe",
	"chatSetSettings",
	"chestBuy",
	"clanGetActivityRewardTable",
	"clanGetActivityStat",
	"clanGetInfo",
	"clanGetPrevData",
	"clanGetWeeklyStat",
	"clanWarAttack",
	"clanWarEndBattle",
	"clanWarGetAvailableHistory",
	"clanWarGetBriefInfo",
	"clanWarGetDayHistory",
	"clanWarGetDefence",
	"clanWarGetInfo",
	"clanWarGetWarlordInfo",
	"consumableUseHeroXp",
	"consumableUseLootBox",
	"consumableUsePetXp",
	"consumableUseStamina",
	"dailyBonusFarm",
	"dailyBonusGetInfo",
	"dungeonEndBattle",
	"dungeonGetInfo",
	"dungeonSaveProgress",
	"dungeonStartBattle",
	"expeditionFarm",
	"expeditionGet",
	"expeditionSendHeroes",
	"freebieCheck",
	"freebieHaveGroup",
	"friendsGetInfo",
	"friendsSendDailyGift",
	"getTime",
	"grandAttack",
	"grandCheckTargetRange",
	"grandFindEnemies",
	"hallOfFameGetTrophies",
	"heroArtifactLevelUp",
	"heroEnchantRune",
	"heroesMerchantGet",
	"heroGetAll",
	"heroSkinUpgrade",
	"heroTitanGiftLevelUp",
	"heroUpgradeSkill",
	"inventoryCraftFragments",
	"inventoryCraftRecipe",
	"inventoryExchangeHeroArtifacts",
	"inventoryExchangeStones",
	"inventoryExchangeTitanStones",
	"inventoryGet",
	"lootBoxBuy",
	"mailFarm",
	"mailGetAll",
	"mechanicAvailability",
	"missionGetAll",
	"missionGetReplace",
	"missionRaid",
	"newYearGetInfo",
	"offerFarmReward",
	"offerGetAll",
	"offerTrialPay",
	"pet_chestOpen",
	"pet_getAll",
	"pet_getChest",
	"pet_getPotionDailyBuyCount",
	"pet_levelUpItem",
	"pirateTreasureIsAvailable",
	"playable_getAvailable",
	"questFarm",
	"questGetAll",
	"questGetEvents",
	"questGetPaymentDependent",
	"refillableAlchemyUse",
	"refillableBuyStamina",
	"registration",
	"settingsGetAll",
	"settingsSet",
	"shopBuy",
	"shopGet",
	"shopGetAll",
	"socialQuestGetInfo",
	"splitGetAll",
	"stashClient",
	"subscriptionFarm",
	"subscriptionGetInfo",
	"teamGetAll",
	"teamGetMaxUpgrade",
	"titanArenaCheckForgotten",
	"titanArenaCompleteTier",
	"titanArenaEndBattle",
	"titanArenaEndRaid",
	"titanArenaFarmDailyReward",
	"titanArenaGetChestReward",
	"titanArenaGetDailyReward",
	"titanArenaGetLog",
	"titanArenaGetStatus",
	"titanArenaStartBattle",
	"titanArenaStartRaid",
	"titanArtifactChestOpen",
	"titanArtifactGetChest",
	"titanArtifactLevelUp",
	"titanArtifactLevelUpStarmoney",
	"titanGetAll",
	"titanGetSummoningCircle",
	"titanSpiritGetAll",
	"titanUseSummonCircle",
	"tower_farmPointRewards",
	"tower_farmSkullReward",
	"towerGetInfo",
	"tower_getSkullReward",
	"towerNextChest",
	"towerOpenChest",
	"tutorialGetInfo",
	"userGetAvailableAvatars",
	"userGetAvailableStickers",
	"userGetInfo",
	"userMergeGetStatus",
	"zeppelinGiftFarm",
	"zeppelinGiftGet",
}

var dataTypesActive = []string{}

// NewPreProcessor ...
func NewPreProcessor(path string, debug bool) *PreProcessor {
	p := new(PreProcessor)
	p.Dir = path
	path, _ = filepath.Abs(path)
	p.Debug = debug
	return p
}

// PreProcessor ...
type PreProcessor struct {
	Dir   string
	Debug bool
}

// RunPath ...
func (p *PreProcessor) RunPath() {
	p.main()
}

// RunBody ...
func (p *PreProcessor) RunBody(data []byte) error {
	return p.processJSONBody(data)
}

// main to run it all...
func (p *PreProcessor) main() {
	fmt.Printf("START PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999"))
	err := filepath.Walk(p.Dir,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !strings.Contains(path, "\\2020-") && !strings.Contains(path, "/2020-") {
				return nil
			}
			if info.IsDir() {
				return nil
			}
			if strings.HasSuffix(info.Name(), ".json") {
				err = p.processJSONFile(path, info)
				if err != nil {
					fmt.Println(err)
				}
			}
			return nil
		})
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("END PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999"))
}

func (p *PreProcessor) processJSONBody(json []byte) error {
	date := gjson.GetBytes(json, `response.body.date`)
	if date.Exists() {
		req := gjson.GetBytes(json, `request`)
		if !req.Exists() {
			return errors.New("no request")
		}
		reqHeaders := req.Get(`headers`)
		if !reqHeaders.Exists() {
			return errors.New("no request.headers")
		}
		for _, identSearch := range dataTypes {

			call := req.Get(`body.calls.#(name=="` + identSearch + `")`)
			if !call.Exists() {
				continue
			}
			ident := call.Get("ident")
			if !ident.Exists() {
				continue
			}

			data := gjson.GetBytes(json, `response.body.results.#(ident=="`+ident.String()+`")`)
			if !data.Exists() {
				data2 := gjson.GetBytes(json, `response.body`)
				if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" {
					return fmt.Errorf("JSON with invalide signature")
				}
				continue
			}

			saveDir := filepath.Join(p.Dir, "data", identSearch)
			saveFile := filepath.Join(saveDir, date.Raw+"_"+reqHeaders.Get("X-Auth-User-Id").String()+".json")
			os.MkdirAll(saveDir, os.ModePerm)

			if _, err := os.Stat(saveFile); !os.IsNotExist(err) {
				continue
			}

			var sdata []byte
			var errdata error
			sdata, errdata = sjson.SetRawBytes(sdata, "date", []byte(date.Raw))
			if errdata != nil {
				return errdata
			}
			sdata, errdata = sjson.SetRawBytes(sdata, "request.headers", []byte(reqHeaders.Raw))
			if errdata != nil {
				return errdata
			}
			sdata, errdata = sjson.SetRawBytes(sdata, "request.call", []byte(call.Raw))
			if errdata != nil {
				return errdata
			}
			sdata, errdata = sjson.SetRawBytes(sdata, "request.ident", []byte(ident.Raw))
			if errdata != nil {
				return errdata
			}
			sdata, errdata = sjson.SetRawBytes(sdata, "response", []byte(data.Raw))
			if errdata != nil {
				return errdata
			}

			err := ioutil.WriteFile(saveFile, sdata, os.ModePerm)
			if err != nil {
				fmt.Printf("%v - %v - %v\n", saveDir, saveFile, err)
			}
		}
	} else {
		data2 := gjson.GetBytes(json, `response.body`)
		if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" {
			return fmt.Errorf("%s: JSON with invalide signature", "bodyData")
		}
	}
	return nil
}

func (p *PreProcessor) processJSONFile(path string, info os.FileInfo) error {

	fmt.Printf("START PreProcessing of %s at %s\n", path, time.Now().UTC().Format("2006-01-02 15:04:05.999999"))
	json, err := p.readJSON(path)
	if err != nil {
		return err
	}

	return p.processJSONBody(json)
}

func (p *PreProcessor) readJSON(path string) ([]byte, error) {
	// read file
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return []byte(""), err
	}
	return data, nil
}