add datapreprocessor + some more data

server/services/cronjob/cronjob.go

@@ -0,0 +1,27 @@
+package cronjob
+
+import (
+	"hwcollector/server/services/datapreprocessor"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/robfig/cron/v3"
+)
+
+// InitCron start cron
+func InitCron(debug bool) *cron.Cron {
+	cj := cron.New(cron.WithSeconds(), cron.WithLocation(time.UTC))
+	cj.AddFunc("1 0 0 * * *", func() {
+		t := time.Now().UTC()
+		newpath := filepath.Join(".", t.Format("2006-01-02"))
+		os.MkdirAll(newpath, os.ModePerm)
+	})
+	cj.AddFunc("1 0/15 * * * *", func() {
+		pp := datapreprocessor.NewPreProcessor(".", debug)
+		pp.RunPath()
+	})
+
+	cj.Start()
+	return cj
+}

server/services/database/database.go

@@ -0,0 +1 @@
+package database

server/services/datacollector/datacollector.go

@@ -0,0 +1 @@
+package datacollector

server/services/datapreprocessor/datapreprocessor.go

@@ -0,0 +1,317 @@
+package datapreprocessor
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+var dataTypes = []string{"adventure_collectReward",
+	"adventure_endBattle",
+	"adventure_find",
+	"adventure_getActiveData",
+	"adventure_getGlobalBuffs",
+	"adventure_getInfo",
+	"adventure_getPassed",
+	"adventure_start",
+	"adventure_turnCollectBuff",
+	"adventure_turnStartBattle",
+	"arenaAttack",
+	"arenaCheckTargetRange",
+	"arenaFindEnemies",
+	"arenaGetAll",
+	"artifactChestOpen",
+	"artifactGetChestLevel",
+	"battleGetByType",
+	"battleGetReplay",
+	"billingGetAll",
+	"billingGetLast",
+	"bossGetAll",
+	"bossOpenChest",
+	"bossRaid",
+	"campaignStoryGetList",
+	"chatAcceptChallenge",
+	"chatGetAll",
+	"chatGetInfo",
+	"chatGetTalks",
+	"chatSendChallenge",
+	"chatSendText",
+	"chatServerSubscribe",
+	"chatServerUnsubscribe",
+	"chatSetSettings",
+	"chestBuy",
+	"clanGetActivityRewardTable",
+	"clanGetActivityStat",
+	"clanGetInfo",
+	"clanGetPrevData",
+	"clanGetWeeklyStat",
+	"clanWarAttack",
+	"clanWarEndBattle",
+	"clanWarGetAvailableHistory",
+	"clanWarGetBriefInfo",
+	"clanWarGetDayHistory",
+	"clanWarGetDefence",
+	"clanWarGetInfo",
+	"clanWarGetWarlordInfo",
+	"consumableUseHeroXp",
+	"consumableUseLootBox",
+	"consumableUsePetXp",
+	"consumableUseStamina",
+	"dailyBonusFarm",
+	"dailyBonusGetInfo",
+	"dungeonEndBattle",
+	"dungeonGetInfo",
+	"dungeonSaveProgress",
+	"dungeonStartBattle",
+	"expeditionFarm",
+	"expeditionGet",
+	"expeditionSendHeroes",
+	"freebieCheck",
+	"freebieHaveGroup",
+	"friendsGetInfo",
+	"friendsSendDailyGift",
+	"getTime",
+	"grandAttack",
+	"grandCheckTargetRange",
+	"grandFindEnemies",
+	"hallOfFameGetTrophies",
+	"heroArtifactLevelUp",
+	"heroEnchantRune",
+	"heroesMerchantGet",
+	"heroGetAll",
+	"heroSkinUpgrade",
+	"heroTitanGiftLevelUp",
+	"heroUpgradeSkill",
+	"inventoryCraftFragments",
+	"inventoryCraftRecipe",
+	"inventoryExchangeHeroArtifacts",
+	"inventoryExchangeStones",
+	"inventoryExchangeTitanStones",
+	"inventoryGet",
+	"lootBoxBuy",
+	"mailFarm",
+	"mailGetAll",
+	"mechanicAvailability",
+	"missionGetAll",
+	"missionGetReplace",
+	"missionRaid",
+	"newYearGetInfo",
+	"offerFarmReward",
+	"offerGetAll",
+	"offerTrialPay",
+	"pet_chestOpen",
+	"pet_getAll",
+	"pet_getChest",
+	"pet_getPotionDailyBuyCount",
+	"pet_levelUpItem",
+	"pirateTreasureIsAvailable",
+	"playable_getAvailable",
+	"questFarm",
+	"questGetAll",
+	"questGetEvents",
+	"questGetPaymentDependent",
+	"refillableAlchemyUse",
+	"refillableBuyStamina",
+	"registration",
+	"settingsGetAll",
+	"settingsSet",
+	"shopBuy",
+	"shopGet",
+	"shopGetAll",
+	"socialQuestGetInfo",
+	"splitGetAll",
+	"stashClient",
+	"subscriptionFarm",
+	"subscriptionGetInfo",
+	"teamGetAll",
+	"teamGetMaxUpgrade",
+	"titanArenaCheckForgotten",
+	"titanArenaCompleteTier",
+	"titanArenaEndBattle",
+	"titanArenaEndRaid",
+	"titanArenaFarmDailyReward",
+	"titanArenaGetChestReward",
+	"titanArenaGetDailyReward",
+	"titanArenaGetLog",
+	"titanArenaGetStatus",
+	"titanArenaStartBattle",
+	"titanArenaStartRaid",
+	"titanArtifactChestOpen",
+	"titanArtifactGetChest",
+	"titanArtifactLevelUp",
+	"titanArtifactLevelUpStarmoney",
+	"titanGetAll",
+	"titanGetSummoningCircle",
+	"titanSpiritGetAll",
+	"titanUseSummonCircle",
+	"tower_farmPointRewards",
+	"tower_farmSkullReward",
+	"towerGetInfo",
+	"tower_getSkullReward",
+	"towerNextChest",
+	"towerOpenChest",
+	"tutorialGetInfo",
+	"userGetAvailableAvatars",
+	"userGetAvailableStickers",
+	"userGetInfo",
+	"userMergeGetStatus",
+	"zeppelinGiftFarm",
+	"zeppelinGiftGet",
+}
+
+// NewPreProcessor ...
+func NewPreProcessor(path string, debug bool) *PreProcessor {
+	p := new(PreProcessor)
+	p.Dir = path
+	path, _ = filepath.Abs(path)
+	fmt.Println(path)
+	p.Debug = debug
+	return p
+}
+
+// PreProcessor ...
+type PreProcessor struct {
+	Dir   string
+	Debug bool
+}
+
+// RunPath ...
+func (p *PreProcessor) RunPath() {
+	p.main()
+}
+
+// RunBody ...
+func (p *PreProcessor) RunBody(data []byte) error {
+	return p.processJSONBody(data)
+}
+
+// main to run it all...
+func (p *PreProcessor) main() {
+	fmt.Printf("START PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999"))
+	err := filepath.Walk(p.Dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !strings.Contains(path, "\\2020-") && !strings.Contains(path, "/2020-") {
+				return nil
+			}
+			if info.IsDir() {
+				return nil
+			}
+			if strings.HasSuffix(info.Name(), ".json") {
+				err = p.processJSONFile(path, info)
+				if err != nil {
+					fmt.Println(err)
+				}
+			}
+			return nil
+		})
+	if err != nil {
+		fmt.Println(err)
+	}
+	fmt.Printf("END PreProcessing: %s\n", time.Now().UTC().Format("2006-01-02 15:04:05.999999999"))
+}
+
+func (p *PreProcessor) processJSONBody(json []byte) error {
+	date := gjson.GetBytes(json, `response.body.date`)
+	if date.Exists() {
+		req := gjson.GetBytes(json, `request`)
+		if !req.Exists() {
+			return errors.New("no request")
+		}
+		reqHeaders := req.Get(`headers`)
+		if !reqHeaders.Exists() {
+			return errors.New("no request.headers")
+		}
+		for _, identSearch := range dataTypes {
+
+			call := req.Get(`body.calls.#(name=="` + identSearch + `")`)
+			if !call.Exists() {
+				continue
+			}
+			ident := call.Get("ident")
+			if !ident.Exists() {
+				continue
+			}
+
+			data := gjson.GetBytes(json, `response.body.results.#(ident=="`+ident.String()+`")`)
+			if !data.Exists() {
+				data2 := gjson.GetBytes(json, `response.body`)
+				if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" {
+					return fmt.Errorf("JSON with invalide signature")
+				}
+				continue
+			}
+
+			saveDir := filepath.Join(p.Dir, "data", identSearch)
+			saveFile := filepath.Join(saveDir, date.Raw+"_"+reqHeaders.Get("X-Auth-User-Id").String()+".json")
+			os.MkdirAll(saveDir, os.ModePerm)
+
+			if _, err := os.Stat(saveFile); !os.IsNotExist(err) {
+				continue
+			}
+
+			var sdata []byte
+			var errdata error
+			sdata, errdata = sjson.SetRawBytes(sdata, "date", []byte(date.Raw))
+			if errdata != nil {
+				return errdata
+			}
+			sdata, errdata = sjson.SetRawBytes(sdata, "request.headers", []byte(reqHeaders.Raw))
+			if errdata != nil {
+				return errdata
+			}
+			sdata, errdata = sjson.SetRawBytes(sdata, "request.call", []byte(call.Raw))
+			if errdata != nil {
+				return errdata
+			}
+			sdata, errdata = sjson.SetRawBytes(sdata, "request.ident", []byte(ident.Raw))
+			if errdata != nil {
+				return errdata
+			}
+			sdata, errdata = sjson.SetRawBytes(sdata, "response", []byte(data.Raw))
+			if errdata != nil {
+				return errdata
+			}
+
+			err := ioutil.WriteFile(saveFile, sdata, os.ModePerm)
+			if err != nil {
+				fmt.Printf("%v - %v - %v\n", saveDir, saveFile, err)
+			}
+		}
+	} else {
+		data2 := gjson.GetBytes(json, `response.body`)
+		if data2.Exists() && data2.String() == "SW52YWxpZCBzaWduYXR1cmU=" {
+			return fmt.Errorf("%s: JSON with invalide signature", "bodyData")
+		}
+	}
+	return nil
+}
+
+func (p *PreProcessor) processJSONFile(path string, info os.FileInfo) error {
+
+	fmt.Printf("START PreProcessing of %s at %s\n", path, time.Now().UTC().Format("2006-01-02 15:04:05.999999"))
+	json, err := p.readJSON(path)
+	if err != nil {
+		return err
+	}
+
+	return p.processJSONBody(json)
+}
+
+func (p *PreProcessor) readJSON(path string) ([]byte, error) {
+	// read file
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return data, err
+	}
+	return data, nil
+}

server/services/datapreprocessor/datapreprocessor_test.go

@@ -0,0 +1,19 @@
+package datapreprocessor
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestMain(t *testing.T) {
+	pathDir, err := os.Getwd()
+	t.Log(pathDir)
+	pathDir = filepath.Join(pathDir, "..", "..", "..")
+	if err != nil {
+		t.Logf("%#v\n", err)
+	}
+	t.Log(pathDir)
+
+	//Main(pathDir)
+}

server/services/services.go

@@ -0,0 +1,36 @@
+package services
+
+import (
+	"hwcollector/server/services/cronjob"
+	"hwcollector/server/services/datapreprocessor"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/robfig/cron/v3"
+)
+
+// Services all Services
+type Services struct {
+	DataPreProcessing *datapreprocessor.PreProcessor
+	CronJob           *cron.Cron
+}
+
+// MainServices Get all Services
+func MainServices(debug bool) *Services {
+	cj := cronjob.InitCron(debug)
+	services := new(Services)
+	services.CronJob = cj
+
+	dir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	path, err := filepath.Abs(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+	services.DataPreProcessing = datapreprocessor.NewPreProcessor(path, debug)
+
+	return services
+}