remove dumps and trash

add some new files and services

add hwtools that shows the current and last GWs - not yet included the user stats (if any available)

server/models/clanWarGetInfo/clanWarGetInfo.go

@@ -0,0 +1,122 @@
+// This file was generated from JSON Schema using quicktype, do not modify it directly.
+// To parse and unparse this JSON data, add this code to your project and do:
+//
+//    clanWarGetInfo, err := UnmarshalClanWarGetInfo(bytes)
+//    bytes, err = clanWarGetInfo.Marshal()
+
+package main
+
+import (
+	"hwcollector/server/models/common"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+func UnmarshalClanWarGetInfo(data []byte) (ClanWarGetInfo, error) {
+	var r ClanWarGetInfo
+	err := json.Unmarshal(data, &r)
+	return r, err
+}
+
+func (r *ClanWarGetInfo) Marshal() ([]byte, error) {
+	return json.Marshal(r)
+}
+
+type ClanWarGetInfo struct {
+	Data     float64                `json:"data"`
+	Headers  common.Headers         `json:"headers"`
+	Response ClanWarGetInfoResponse `json:"response"`
+}
+
+type ClanWarGetInfoResponse struct {
+	Ident  string `json:"ident"`
+	Result Result `json:"result"`
+}
+
+type Result struct {
+	Response ResultResponse `json:"response"`
+}
+
+type ResultResponse struct {
+	ArePointsMax     bool                       `json:"arePointsMax"`
+	AvgLevel         string                     `json:"avgLevel"`
+	ClanTries        map[string]int64           `json:"clanTries"`
+	Day              string                     `json:"day"`
+	EndTime          int64                      `json:"endTime"`
+	EnemyClan        EnemyClan                  `json:"enemyClan"`
+	EnemyClanMembers map[string]EnemyClanMember `json:"enemyClanMembers"`
+	EnemyClanTries   map[string]int64           `json:"enemyClanTries"`
+	EnemyID          string                     `json:"enemyId"`
+	EnemyPoints      string                     `json:"enemyPoints"`
+	EnemySlots       map[string]Slot            `json:"enemySlots"`
+	League           string                     `json:"league"`
+	MyTries          int64                      `json:"myTries"`
+	NextLockTime     int64                      `json:"nextLockTime"`
+	NextWarTime      int64                      `json:"nextWarTime"`
+	OurSlots         map[string]Slot            `json:"ourSlots"`
+	Points           string                     `json:"points"`
+	Season           string                     `json:"season"`
+}
+
+type EnemyClan struct {
+	Country      string        `json:"country"`
+	Description  string        `json:"description"`
+	Disbanding   bool          `json:"disbanding"`
+	FrameID      int64         `json:"frameId"`
+	ID           string        `json:"id"`
+	Level        string        `json:"level"`
+	MembersCount string        `json:"membersCount"`
+	MinLevel     string        `json:"minLevel"`
+	OwnerID      string        `json:"ownerId"`
+	RoleNames    []interface{} `json:"roleNames"`
+	ServerID     string        `json:"serverId"`
+	Title        string        `json:"title"`
+	TopActivity  string        `json:"topActivity"`
+	TopDungeon   string        `json:"topDungeon"`
+}
+
+type EnemyClanMember struct {
+	AllowPm         string `json:"allowPm"`
+	AvatarID        string `json:"avatarId"`
+	ClanID          string `json:"clanId"`
+	ClanRole        string `json:"clanRole"`
+	ClanTitle       string `json:"clanTitle"`
+	FrameID         int64  `json:"frameId"`
+	ID              string `json:"id"`
+	IsChatModerator bool   `json:"isChatModerator"`
+	LastLoginTime   string `json:"lastLoginTime"`
+	LeagueID        int64  `json:"leagueId"`
+	Level           string `json:"level"`
+	Name            string `json:"name"`
+	ServerID        string `json:"serverId"`
+}
+
+type Slot struct {
+	AttackerID          int64             `json:"attackerId"`
+	PointsFarmed        int64             `json:"pointsFarmed"`
+	SlotID              int64             `json:"slotId"`
+	Status              string            `json:"status"`
+	TargetMarkingUserID *int64            `json:"targetMarkingUserId,omitempty"`
+	Team                []map[string]Team `json:"team"`
+	TotalPoints         int64             `json:"totalPoints"`
+	User                EnemyClanMember   `json:"user"`
+}
+
+type Team struct {
+	Color *int64 `json:"color,omitempty"`
+	ID    int64  `json:"id"`
+	Level int64  `json:"level"`
+	Power int64  `json:"power"`
+	Star  int64  `json:"star"`
+	State State  `json:"state"`
+	Type  string `json:"type"`
+}
+
+type State struct {
+	Energy int64 `json:"energy"`
+	HP     int64 `json:"hp"`
+	IsDead bool  `json:"isDead"`
+	MaxHP  int64 `json:"maxHp"`
+}

server/models/common/headers.go

@@ -0,0 +1,17 @@
+package common
+
+type Headers struct {
+	ContentType        string `json:"Content-Type"`
+	XAuthApplicationID string `json:"X-Auth-Application-Id"`
+	XAuthNetworkIdent  string `json:"X-Auth-Network-Ident"`
+	XAuthPlayerID      string `json:"X-Auth-Player-Id"`
+	XAuthSessionID     string `json:"X-Auth-Session-Id"`
+	XAuthSessionKey    string `json:"X-Auth-Session-Key"`
+	XAuthSignature     string `json:"X-Auth-Signature"`
+	XAuthToken         string `json:"X-Auth-Token"`
+	XAuthUserID        string `json:"X-Auth-User-Id"`
+	XEnvLibraryVersion string `json:"X-Env-Library-Version"`
+	XRequestID         string `json:"X-Request-Id"`
+	XRequestedWith     string `json:"X-Requested-With"`
+	XServerTime        string `json:"X-Server-Time"`
+}

server/models/qtmodels/qtmodels_test.go

@@ -0,0 +1,108 @@
+// This file was generated from JSON Schema using quicktype, do not modify it directly.
+// To parse and unparse this JSON data, add this code to your project and do:
+//
+//    qtmodels, err := UnmarshalQtmodels(bytes)
+//    bytes, err = qtmodels.Marshal()
+
+package qtmodels
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestUnmarshalQtmodels(t *testing.T) {
+	defaultPath, _ := os.Getwd()
+	defaultPath, _ = filepath.Abs(defaultPath)
+	defaultPath = filepath.Join(defaultPath, "..", "..", "..", "data")
+	fs := []string{
+		defaultPath + "/arenaAttack_merged.json",
+	}
+
+	for _, v := range fs {
+		t.Run("json: "+string(v), func(t *testing.T) {
+
+			b, err := ioutil.ReadFile(v)
+			if err != nil {
+				t.Errorf("ioutil.ReadFile() error = %v", err)
+				return
+			}
+
+			qtmodels, err := UnmarshalQtmodels(b)
+			if err != nil {
+				t.Errorf("UnmarshalQtmodels() error = %v", err)
+				return
+			}
+			for _, qtmodel := range qtmodels {
+				if qtmodel.Request.Call.Name != "arenaAttack" {
+					t.Error(`qtmodel.Request.Call.Name != "arenaAttack"`)
+					return
+				}
+			}
+			//t.Logf("%#v", qtmodels)
+		})
+	}
+}
+
+func TestUnmarshalQtmodel(t *testing.T) {
+	defaultPath, _ := os.Getwd()
+	defaultPath, _ = filepath.Abs(defaultPath)
+	defaultPath = filepath.Join(defaultPath, "..", "..", "..", "data", "arenaAttack")
+	var fs []string
+
+	err := filepath.Walk(defaultPath, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			fs = append(fs, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Errorf("Walk err = %v", err)
+		return
+	}
+
+	for _, v := range fs {
+		t.Run("json: "+string(v), func(t *testing.T) {
+
+			b, err := ioutil.ReadFile(v)
+			if err != nil {
+				t.Errorf("ioutil.ReadFile() error = %v", err)
+				return
+			}
+
+			qtmodel, err := UnmarshalQtmodel(b)
+			if err != nil {
+				t.Errorf("UnmarshalQtmodel() error = %v", err)
+				return
+			}
+			if qtmodel.Request.Call.Name != "arenaAttack" {
+				t.Error(`qtmodel.Request.Call.Name != "arenaAttack"`)
+				return
+			}
+
+			if qtmodel.Response.Result.Response != nil {
+				if qtmodel.Response.Result.Response.Battles != nil {
+					//t.Logf("%#v", qtmodel.Response.Result.Response.Battles)
+					if len(qtmodel.Response.Result.Response.Battles) != 1 {
+						t.Errorf(`len(qtmodel.Response.Result.Response.Battles) != 6 got %v`, len(qtmodel.Response.Result.Response.Battles))
+						return
+					}
+					for _, battle := range qtmodel.Response.Result.Response.Battles {
+						if len(battle.Attackers) == 0 {
+							t.Errorf(`range qtmodel.Response.Result.Response.Battles: battle.Attackers > 0 got %v`, len(battle.Attackers))
+							return
+						}
+						if len(battle.Defenders) == 0 {
+							t.Errorf(`range qtmodel.Response.Result.Response.Battles: battle.Defenders > 0 got %v`, len(battle.Defenders))
+							return
+						}
+
+					}
+				}
+				//t.Logf("%#v", qtmodels)
+			}
+		})
+	}
+}

server/models/users.go

@@ -0,0 +1,70 @@
+package models
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+// UnmarshalUserJSON ...
+func (u *User) UnmarshalUserJSON(data []byte) error {
+	var f interface{}
+	err := json.Unmarshal(data, &f)
+	if err != nil {
+		return err
+	}
+	m := f.(map[string]interface{})
+	for k, v := range m {
+		switch k {
+		case "allowPm":
+			u.AllowPm = v.(string)
+		case "avatarId":
+			u.AvatarID = v.(string)
+		case "clanId":
+			u.ClanID, _ = strconv.ParseUint(v.(string), 10, 64)
+		case "clanRole":
+			u.ClanRole = v.(string)
+		case "clanTitle":
+			u.ClanTitle = v.(string)
+		case "id":
+			u.ID, _ = strconv.ParseUint(v.(string), 10, 64)
+		case "isChatModerator":
+			u.IsChatModerator = v.(bool)
+		case "lastLoginTime":
+			unixTimeStamp, _ := strconv.ParseInt(v.(string), 10, 64)
+			u.LastLoginTime = time.Unix(unixTimeStamp, 0)
+		case "leagueId":
+			var leagueID *int64
+			if v != nil {
+				l := int64(v.(float64))
+				leagueID = &l
+			}
+			u.LeagueID = leagueID
+		case "level":
+			u.Level = v.(string)
+		case "name":
+			u.Name = v.(string)
+		case "serverId":
+			u.ServerID = v.(string)
+		}
+	}
+	return nil
+}
+
+// User ...
+// used in
+//	- arenaAttack
+type User struct {
+	ID              uint64    `json:"id"`
+	AllowPm         string    `json:"allowPm"`
+	AvatarID        string    `json:"avatarId"`
+	ClanID          uint64    `json:"clanId"`
+	ClanRole        string    `json:"clanRole"`
+	ClanTitle       string    `json:"clanTitle"`
+	IsChatModerator bool      `json:"isChatModerator"`
+	LastLoginTime   time.Time `json:"lastLoginTime"`
+	LeagueID        *int64    `json:"leagueId"`
+	Level           string    `json:"level"`
+	Name            string    `json:"name"`
+	ServerID        string    `json:"serverId"`
+}

server/services/cronjob/cronjob.go

@@ -1,6 +1,8 @@
 package cronjob
 
 import (
+	"fmt"
+	"hwcollector/server/services/datacollector"
 	"hwcollector/server/services/datapreprocessor"
 	"os"
 	"path/filepath"
@@ -14,7 +16,7 @@ func InitCron(debug bool) *cron.Cron {
 	cj := cron.New(cron.WithSeconds(), cron.WithLocation(time.UTC))
 	cj.AddFunc("1 0 0 * * *", func() {
 		t := time.Now().UTC()
-		newpath := filepath.Join(".", t.Format("2006-01-02"))
+		newpath := filepath.Join(".", "_data_", t.Format("2006-01-02"))
 		os.MkdirAll(newpath, os.ModePerm)
 	})
 	cj.AddFunc("1 0/15 * * * *", func() {
@@ -22,6 +24,10 @@ func InitCron(debug bool) *cron.Cron {
 		pp.RunPath()
 	})
 
+	cj.AddFunc("30 1/5 * * * *", func() {
+		fmt.Println("datacollector.Start()")
+		datacollector.Start()
+	})
 	cj.Start()
 	return cj
 }

server/services/datacollector/arenaAttack.go

@@ -0,0 +1,45 @@
+package datacollector
+
+import (
+	"hwcollector/server/models"
+
+	"github.com/tidwall/gjson"
+)
+
+func init() {
+	processes["arenaAttack"] = new(DoArenaAttack)
+}
+
+// DoArenaAttack ...
+type DoArenaAttack struct {
+	data []byte
+}
+
+// Start ...
+func (x *DoArenaAttack) Start() {
+}
+
+// SetJSONByte ...
+func (x *DoArenaAttack) SetJSONByte(b []byte) {
+	x.data = b
+}
+
+// SetJSON ...
+func (x *DoArenaAttack) SetJSON(s string) {
+	x.data = []byte(s)
+}
+
+// GetUserDetails ...
+func (x *DoArenaAttack) GetUserDetails() ([]*models.User, error) {
+	var users []*models.User
+
+	result := gjson.GetBytes(x.data, "response.result.response.enemies.#.user")
+	for _, jsonValue := range result.Array() {
+		user := new(models.User)
+		if err := user.UnmarshalUserJSON([]byte(jsonValue.Raw)); err != nil {
+			return nil, err
+		}
+		users = append(users, user)
+	}
+	return users, nil
+}

server/services/datacollector/arenaFindEnemies.go

@@ -0,0 +1,45 @@
+package datacollector
+
+import (
+	"hwcollector/server/models"
+
+	"github.com/tidwall/gjson"
+)
+
+func init() {
+	processes["arenaFindEnemies"] = new(DoArenaFindEnemies)
+}
+
+// DoArenaFindEnemies ...
+type DoArenaFindEnemies struct {
+	data []byte
+}
+
+// Start ...
+func (x *DoArenaFindEnemies) Start() {
+}
+
+// SetJSONByte ...
+func (x *DoArenaFindEnemies) SetJSONByte(b []byte) {
+	x.data = b
+}
+
+// SetJSON ...
+func (x *DoArenaFindEnemies) SetJSON(s string) {
+	x.data = []byte(s)
+}
+
+// GetUserDetails ...
+func (x *DoArenaFindEnemies) GetUserDetails() ([]*models.User, error) {
+	var users []*models.User
+
+	result := gjson.GetBytes(x.data, "response.result.response.enemies.#.user")
+	for _, jsonValue := range result.Array() {
+		user := new(models.User)
+		if err := user.UnmarshalUserJSON([]byte(jsonValue.Raw)); err != nil {
+			return nil, err
+		}
+		users = append(users, user)
+	}
+	return users, nil
+}

server/services/datacollector/clanWarGetInfo.go

@@ -0,0 +1,140 @@
+package datacollector
+
+import (
+	"fmt"
+	"hwcollector/server/models"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+func init() {
+	processes["clanWarGetInfo"] = new(DoClanWarGetInfo)
+}
+
+const (
+	IdentClanWarGetInfo = "clanWarGetInfo"
+)
+
+// DoClanWarGetInfo ...
+type DoClanWarGetInfo struct {
+	data   []byte
+	result []byte
+}
+
+// Start ...
+func (x *DoClanWarGetInfo) Start() {
+}
+
+// SetJSONByte ...
+func (x *DoClanWarGetInfo) SetJSONByte(b []byte) {
+	x.data = b
+}
+
+// SetJSON ...
+func (x *DoClanWarGetInfo) SetJSON(s string) {
+	x.data = []byte(s)
+}
+
+// GetUserDetails ...
+func (x *DoClanWarGetInfo) GetUserDetails() ([]*models.User, error) {
+	var users []*models.User
+
+	result := gjson.GetBytes(x.data, "response.result.response.enemies.#.user")
+	for _, jsonValue := range result.Array() {
+		user := new(models.User)
+		if err := user.UnmarshalUserJSON([]byte(jsonValue.Raw)); err != nil {
+			return nil, err
+		}
+		users = append(users, user)
+	}
+	return users, nil
+}
+
+// GetObject ...
+func (x *DoClanWarGetInfo) GetObject() []byte {
+	date := gjson.GetBytes(x.data, `date`)
+	if !date.Exists() {
+		fmt.Printf("GetObject - error: %s\n", "can't find date")
+		return []byte("")
+	}
+
+	dat := gjson.GetBytes(x.data, `response.result.response`)
+	if !dat.Exists() {
+		fmt.Printf("GetObject - error: %s\n", "can't find results")
+		return []byte("")
+	}
+
+	warEnd := dat.Get("endTime")
+
+	ts, err := strconv.Atoi(warEnd.String())
+	if err != nil {
+		return []byte("")
+	}
+	tsunix := time.Unix(int64(ts), 0)
+	tsunix = tsunix.Add(-12 * time.Hour)
+	dat2, err := sjson.SetRawBytes([]byte(dat.Raw), "warDate", []byte(fmt.Sprintf("%d", tsunix.Unix())))
+	if err != nil {
+		fmt.Printf("GetObject - error: %s\n", err.Error())
+		return []byte("")
+	}
+
+	return dat2
+}
+
+// GenerateArrayFromDir ...
+func (x *DoClanWarGetInfo) GenerateArrayFromDir(ident string) {
+	dirPath, _ := os.Getwd()
+	dirPathHWtools := filepath.Join(dirPath, "hwtools", "data", ident)
+	dirPath = filepath.Join(dirPath, "data", ident)
+	//	fmt.Printf("%s\n", dirPath)
+	returnData := []byte("[]")
+
+	err := filepath.Walk(dirPath,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			//fmt.Printf("%s\n", path)
+			if strings.HasSuffix(info.Name(), ".json") {
+				x.data, err = readFile(path)
+				if err != nil {
+					fmt.Println(err)
+				}
+
+				returnData, err = sjson.SetRawBytes(returnData, "-1", x.GetObject())
+				if err != nil {
+					fmt.Println(err)
+				}
+			}
+			return nil
+		})
+	if err != nil {
+		fmt.Println(err)
+	}
+	mdir := filepath.Base(dirPathHWtools)
+	os.MkdirAll(mdir, os.ModePerm)
+
+	returnData = runJQUniqe(returnData)
+
+	err = ioutil.WriteFile(dirPathHWtools+".json", returnData, 0644)
+	if err != nil {
+		fmt.Println(err)
+	}
+	return
+
+}
+
+// GetJSONResult ...
+func (x *DoClanWarGetInfo) GetJSONResult() []byte {
+	return x.result
+}

server/services/datacollector/datacollector.go

@@ -1 +1,137 @@
 package datacollector
+
+import (
+	"errors"
+	"fmt"
+	"hwcollector/server/models"
+	"io/ioutil"
+	"log"
+	"os/exec"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+// ErrNotExist path not found
+var ErrNotExist = errors.New("The Path not exist")
+
+// TypeInterface for all the registred collectors
+type TypeInterface interface {
+
+	// Set Data
+	SetJSON(string)
+	SetJSONByte([]byte)
+
+	// Start Processing
+	Start()
+
+	// GetUserDetails
+	// Collects the user details from the JSON report and prepares them for further processing.
+	GetUserDetails() ([]*models.User, error)
+}
+
+var processes = make(map[string]TypeInterface)
+
+// Start ...
+func Start() {
+	fmt.Println("Datacollector.Start started")
+	for x, y := range processes {
+		fmt.Printf("Process: %s\n", x)
+		m, ok := y.(interface{ GenerateArrayFromDir(string) })
+		if ok {
+			fmt.Printf("Process: %s GenerateArrayFromDir\n", x)
+			m.GenerateArrayFromDir(IdentClanWarGetInfo)
+		}
+		_ = x
+
+	}
+}
+
+func GetDataFor(name string, json []byte) (returnData []byte, err error) {
+	req := gjson.GetBytes(json, `request`)
+	if !req.Exists() {
+		return []byte{}, fmt.Errorf("error %d: %v", 1, err)
+	}
+
+	returnData, err = helperGetSetData("response.body.date", "data", json, returnData)
+	if err != nil {
+		return []byte{}, fmt.Errorf("error %d: %v", 2, err)
+	}
+
+	returnData, err = helperGetSetData("request.headers", "headers", json, returnData)
+	if err != nil {
+		return []byte{}, fmt.Errorf("error %d: %v", 3, err)
+	}
+
+	call := req.Get(`body.calls.#(name=="` + name + `")`)
+	if !call.Exists() {
+		return []byte{}, fmt.Errorf("error %d: %v", 4, err)
+	}
+	ident := call.Get("ident")
+	if !ident.Exists() {
+		return []byte{}, fmt.Errorf("error %d: %v", 5, err)
+	}
+
+	returnData, err = helperGetSetData(`response.body.results.#(ident=="`+ident.String()+`")`, "response", json, returnData)
+	if err != nil {
+		return []byte{}, fmt.Errorf("error %d: %v", 6, err)
+	}
+	return
+}
+
+func helperGetSetData(getPath, setObj string, json, jsonOut []byte) (returnData []byte, err error) {
+	// read & set DATE
+	data := gjson.GetBytes(json, getPath)
+	if !data.Exists() {
+		return []byte{}, ErrNotExist
+
+	}
+	returnData, err = sjson.SetBytes(jsonOut, setObj, data.Value())
+	if err != nil {
+		return []byte{}, err
+	}
+	return
+
+}
+
+func helperGetIdent(data []byte, name string) (string, error) {
+
+	call := gjson.GetBytes(data, `request.call`)
+	if !call.Exists() {
+		return "", fmt.Errorf("helperGetIdent error: %s", fmt.Sprintf("request.calls with name '%s' not found", name))
+	}
+	ident := call.Get("ident")
+	if !ident.Exists() {
+		return "", fmt.Errorf("helperGetIdent error: %s", "ident not found")
+	}
+	return ident.String(), nil
+}
+
+func readFile(path string) ([]byte, error) {
+	// read file
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return []byte(""), err
+	}
+	return data, nil
+}
+
+func runJQUniqe(in []byte) []byte {
+	cmd := exec.Command("jq", "unique | unique_by(.warDate)|sort_by(.warDate)|reverse|.[:25]")
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	go func() {
+		defer stdin.Close()
+		stdin.Write(in)
+	}()
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return out
+}

server/services/datapreprocessor/datapreprocessor.go

@@ -166,12 +166,13 @@ var dataTypes = []string{"adventure_collectReward",
 	"zeppelinGiftGet",
 }
 
+var dataTypesActive = []string{}
+
 // NewPreProcessor ...
 func NewPreProcessor(path string, debug bool) *PreProcessor {
 	p := new(PreProcessor)
 	p.Dir = path
 	path, _ = filepath.Abs(path)
-	fmt.Println(path)
 	p.Debug = debug
 	return p
 }
@@ -311,7 +312,7 @@ func (p *PreProcessor) readJSON(path string) ([]byte, error) {
 	// read file
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return data, err
+		return []byte(""), err
 	}
 	return data, nil
 }

server/services/services.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"hwcollector/server/services/cronjob"
+	"hwcollector/server/services/datacollector"
 	"hwcollector/server/services/datapreprocessor"
 	"log"
 	"os"
@@ -14,6 +15,7 @@ import (
 type Services struct {
 	DataPreProcessing *datapreprocessor.PreProcessor
 	CronJob           *cron.Cron
+	DataCollector     func()
 }
 
 // MainServices Get all Services
@@ -31,6 +33,6 @@ func MainServices(debug bool) *Services {
 		log.Fatal(err)
 	}
 	services.DataPreProcessing = datapreprocessor.NewPreProcessor(path, debug)
-
+	services.DataCollector = datacollector.Start
 	return services
 }