commit c9cd2683663e1d7e55caea6febf0c716125216c9 Author: Josef Fröhle Date: Wed Jan 15 13:35:30 2020 +0100 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3eb9230 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +content/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4f501a7 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# Dependencies +- libxml2-dev libonig-dev +- chrome / chromium browser + +# Config + +The Config File `config.yaml` should be located in /etc/fbbot. + +```yaml +development_mode: true +cronjob_interval: "@every 5m" +no_run_on_start: false +http_useragent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 + (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19 +webhooks: + live: + - https... + - https + devel: + - https... + - https +comment_on_posts: +- TY +- ty! +- danke! +- I could use it! +- thx! +- Thank you! +- Merci +- Yeah! +- Vielen Dank! + +``` \ No newline at end of file diff --git a/closehandle.go b/closehandle.go new file mode 100644 index 0000000..76dd77a --- /dev/null +++ b/closehandle.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +// SetupCloseHandler creates a 'listener' on a new goroutine which will notify the +// program if it receives an interrupt from the OS. We then handle this by calling +// our clean up procedure and exiting the program. +func SetupCloseHandler() { + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\r- Ctrl+C pressed in Terminal") + os.Exit(0) + }() +} diff --git a/configuration.go b/configuration.go new file mode 100644 index 0000000..2b9b3f1 --- /dev/null +++ b/configuration.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/chromedp/cdproto/cdp" + "github.com/spf13/viper" +) + +const ( + emailField = `//*[@id="m_login_email"]` + passwordField = `//*[@id="m_login_password"]` +) + +var ( + headers = map[string]interface{}{ + "Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7", + } + nodes []*cdp.Node + + randomText = viper.GetStringSlice("comment_on_posts") +) diff --git a/content_functions.go b/content_functions.go new file mode 100644 index 0000000..d2a7ea9 --- /dev/null +++ b/content_functions.go @@ -0,0 +1,10 @@ +package main + +import "strings" + +func getImageURL(testStyleURL string) (string, error) { + genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`) + genURL = strings.ReplaceAll(genURL, `\3d `, `=`) + genURL = strings.ReplaceAll(genURL, `\26 `, `&`) + return genURL, nil +} diff --git a/facebook.go b/facebook.go new file mode 100644 index 0000000..906518b --- /dev/null +++ b/facebook.go @@ -0,0 +1,691 @@ +package main + +import ( + "bufio" + "errors" + "html" + "net/url" + "regexp" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/jbowtie/gokogiri" + gokogirixml "github.com/jbowtie/gokogiri/xml" + "github.com/lunny/html2md" + "github.com/microcosm-cc/bluemonday" + "mvdan.cc/xurls/v2" +) + +// xPath constants +const ( + xPathArticle string = `(//*/article[contains(concat(" ", normalize-space(@class)," "), " _55wo ") and contains(concat(" ", normalize-space(@class)," "), " _5rgr ")])[1]` + xPathArticleContent string = xPathArticle + `/div/div[contains(concat(" ", normalize-space(@class)," "), " _5rgt ") and contains(concat(" ", normalize-space(@class)," "), " _5nk5 ")]/span` + xPathImageURL string = xPathArticle + `/div/div[2]/div[1]/a/div/div/i` + xPathImageURL2 string = xPathArticle + `/div/div[2]/section/div/i` + xPathImageURL3 string = xPathArticle + `/div/div[2]/section/div/div/i` // Video Preview + xPathGiftURL string = xPathArticle + `/div/div[2]/section/a` + xPathPostingURL string = xPathArticle + `/div/header/div[2]/div/div/div[1]/div/a` +) + +// internal URL constants +const ( + fbPageURL string = "https://www.facebook.com/herowarsgame/" + webhookLive string = "https://discordapp.com/api/webhooks/..." + webhookLiveExcelsior string = "https://discordapp.com/api/webhooks/..." + webhookDev string = "https://discordapp.com/api/webhooks/..." + + // fbGameURL for search on expaned urls + fbGameURL string = "apps.facebook.com/mobaheroes" +) + +var ( + dataFT *DataFT + + lastUpdatePosted string + regexStyleImage = regexp.MustCompile(`(?m)url\((.*)\);`) +) + +// Constants for all search for title of the Postings +const ( + FreeTitanArtifact string = "FREE Titan Artifact" + FreeSilverCaskets string = "FREE Silver Caskets" + FreeSoulStones string = "Soul Stones" + FreeSkinStones string = "Skin Stones" + ActionKeepTheAmount string = "Keep the amount" + FreeWinterfestBaubles string = "Winterfest Baubles" + FreeTopFanPackage string = "Top Fan" + FreeEnergyForFee string = "ENERGY FOR FREE" + WinterfestRankingRewards string = "Winterfest ranking rewards" +) + +//FBPostData FBPostData +type FBPostData struct { + PostURL string + TimeStamp string + ProfileLink *ProfileLink + GiftURL string + ImageURL string + Content string + Summary string + Title string + Author string + Tags string +} + +// ParsePost ParsePost +func ParsePost(s, PostURL string) (*FBPostData, error) { + fb := FBPostData{PostURL: PostURL} + + docKogiri, err := gokogiri.ParseHtml([]byte(s)) + if err != nil { + return &fb, err + } + defer docKogiri.Free() + htmlNode := docKogiri.Root().FirstChild() + + //doc, err := goquery.NewDocumentFromReader(strings.NewReader(s)) + //if err != nil { + // return &fb, err + //} + + fb.TimeStamp, err = GetTimeStamp(htmlNode) + if err != nil { + return &fb, err + } + + //fmt.Printf("\n\n%#v\n\n", dataFT) + + //fb.ProfileLink, err = GetProfileLink(doc) + //if err != nil { + // return &fb, err + //} + + fb.ImageURL, err = GetImageURL(htmlNode) + if err != nil { + return &fb, err + } + + fb.GiftURL, err = GetGiftURL(htmlNode) + if err != nil { + return &fb, err + } + + fb.PostURL, err = GetPostURL(htmlNode) + if err != nil { + fb.PostURL = PostURL + return &fb, err + } + + fb.Content, err = GetContent(htmlNode) + if err != nil { + return &fb, err + } + + if strings.Contains(fb.Content, FreeSilverCaskets) { + if len(fb.Title) == 0 { + fb.Title = FreeSilverCaskets + } else { + fb.Title = FreeSilverCaskets + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, FreeTitanArtifact) { + if len(fb.Title) == 0 { + fb.Title = FreeTitanArtifact + } else { + fb.Title = FreeTitanArtifact + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, FreeSoulStones) { + if len(fb.Title) == 0 { + fb.Title = FreeSoulStones + } else { + fb.Title = FreeSoulStones + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, FreeSkinStones) { + if len(fb.Title) == 0 { + fb.Title = FreeSkinStones + } else { + fb.Title = FreeSkinStones + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, ActionKeepTheAmount) { + if len(fb.Title) == 0 { + fb.Title = ActionKeepTheAmount + } else { + fb.Title = ActionKeepTheAmount + "+" + fb.Title + } + fb.GiftURL = fb.PostURL + } + + if strings.Contains(fb.Content, FreeWinterfestBaubles) { + if len(fb.Title) == 0 { + fb.Title = FreeWinterfestBaubles + } else { + fb.Title = FreeWinterfestBaubles + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, FreeTopFanPackage) { + if len(fb.Title) == 0 { + fb.Title = FreeTopFanPackage + } else { + fb.Title = FreeTopFanPackage + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, FreeEnergyForFee) { + if len(fb.Title) == 0 { + fb.Title = FreeEnergyForFee + } else { + fb.Title = FreeEnergyForFee + "+" + fb.Title + } + } + + if strings.Contains(fb.Content, WinterfestRankingRewards) { + if len(fb.Title) == 0 { + fb.Title = WinterfestRankingRewards + } else { + fb.Title = WinterfestRankingRewards + "+" + fb.Title + } + } + + if len(fb.Title) == 0 { + fb.Title = "unknown - need to implemented" + } + defer func() { + dataFT = nil + }() + + return &fb, nil +} + +// Parse Parse +func Parse(url string) (*FBPostData, error) { + doc, err := goquery.NewDocument(url) + if err != nil { + return nil, err + } + + if strings.Contains(url, ".blogspot.") { + return ParseBlogspotPost(doc) + } + + // If not login, post looks like + //
+ s := QuerySelector(doc, "div.hidden_elem > code") + cmt, err := s.Html() + if err != nil { + return nil, err + } + if len(cmt) != 0 { + return ParsePost(cmt, url) + } + + s = QuerySelector(doc, "div._427x") + cmt, err = s.Html() + if err != nil { + return nil, err + } + return ParsePost(cmt, url) +} + +// ParseAll ParseAll +func ParseAll(url string) ([]*FBPostData, error) { + doc, err := goquery.NewDocument(url) + if err != nil { + return nil, err + } + + allFbPosts := []*FBPostData{} + + QuerySelectorEach(doc, "div._427x", func(i int, selected *goquery.Selection) { + cmt, err := selected.Html() + if err != nil { + return + } + fbPost, err := ParsePost(cmt, url) + if err != nil { + return + } + allFbPosts = append(allFbPosts, fbPost) + }) + + return allFbPosts, nil +} + +// GetContent GetContent +func GetContent(htmlNode gokogirixml.Node) (string, error) { + results, err := htmlNode.Search(xPathArticleContent) + if err != nil { + //fmt.Printf("ERR: %#v -- %#v\n", results, err) + return "", err + } + //fmt.Printf("ERR: %#v -- %#v\n", results, err) + if len(results) > 0 { + if resultHTML := results[0].InnerHtml(); resultHTML != "" { + + bmsanizer := bluemonday.StrictPolicy() + bmsanizer.AllowAttrs("href").OnElements("a") + bmsanizer.AllowElements("p") + bmsanizer.RequireParseableURLs(true) + htmlText := bmsanizer.SanitizeBytes([]byte(resultHTML)) + + content := strings.Join(strings.Fields(string(htmlText)), " ") + content = html2md.Convert(content) + content = html.UnescapeString(content) + content = strings.TrimSpace(content) + + return content, nil + } + } + return "", errors.New("no content found") +} + +// GetImageURL GetImageURL +func GetImageURL(htmlNode gokogirixml.Node) (string, error) { + + results, err := htmlNode.Search(xPathImageURL) + if err != nil { + //fmt.Printf("ERR: %#v -- %#v\n", results, err) + return "", err + } + //fmt.Printf("RESULT: %#v -- %#v\n", results, err) + // search for the second possible image + if len(results) == 0 { + results, err = htmlNode.Search(xPathImageURL2) + if err != nil { + //fmt.Printf("ERR2: %#v -- %#v\n", results, err) + return "", err + } + } + // Video Preview + if len(results) == 0 { + results, err = htmlNode.Search(xPathImageURL3) + if err != nil { + //fmt.Printf("ERR2: %#v -- %#v\n", results, err) + return "", err + } + } + //fmt.Printf("RESULT2: %#v -- %#v\n", results, err) + if len(results) > 0 { + if attrib := results[0].Attribute("style"); attrib != nil { + styleContent := attrib.Value() + codedImageURL := regexStyleImage.FindString(styleContent) + genURL := strings.ReplaceAll(codedImageURL, `\3a `, `:`) + genURL = strings.ReplaceAll(genURL, `\3d `, `=`) + genURL = strings.ReplaceAll(genURL, `\26 `, `&`) + genURL = strings.ReplaceAll(genURL, `\25 `, `%`) + genURL = strings.TrimPrefix(genURL, `url('`) + genURL = strings.TrimSuffix(genURL, `');`) + return genURL, nil + + } + } + return "", errors.New("cannot find image url") + //s := QuerySelector(doc, "img.scaledImageFitHeight") + //if s.Length() == 0 { + // s = QuerySelector(doc, "img.scaledImageFitWidth") + //} + // + //url, ok := s.Attr("src") + //if !ok { + // return "", errors.New("cannot find image url") + //} + // + //return url, nil +} + +// GetGiftURL GetGiftURL +func GetGiftURL(htmlNode gokogirixml.Node) (string, error) { + + //s := QuerySelector(doc, "div._6ks > a") + // + //url, ok := s.Attr("href") + + results, err := htmlNode.Search(xPathGiftURL) + if err != nil { + // search for the second possible image + results, err = htmlNode.Search(xPathImageURL2) + if err != nil { + return "", err + } + } + + var genURL string + var attrib *gokogirixml.AttributeNode + if len(results) > 0 { + attrib = results[0].Attribute("href") + if attrib != nil { + tmpGiftURL := attrib.Value() + tmpGiftURL = html.UnescapeString(tmpGiftURL) + tmpGiftURL2, err := url.Parse(tmpGiftURL) + if err != nil { + attrib = nil + goto nextGiftLinkChecker + } + + //fmt.Printf("%#v\n", tmpGiftURL2) + + newQuery := url.Values{} + oldQuery := tmpGiftURL2.Query() + newQuery.Set("nx_source", oldQuery.Get("nx_source")) + newQuery.Set("gift_id", oldQuery.Get("gift_id")) + + tmpURLGen := url.URL{ + Scheme: tmpGiftURL2.Scheme, + Host: tmpGiftURL2.Host, + Path: tmpGiftURL2.Path, + RawQuery: newQuery.Encode(), + } + if tmpURLGen.Scheme != "https" { + tmpURLGen.Scheme = "https" + } + genURL = tmpURLGen.String() + } + } +nextGiftLinkChecker: + if attrib == nil { + results, err := htmlNode.Search(xPathArticleContent) + if err != nil { + return "", err + } + if len(results) > 0 { + //TODO: Fix and check all URLs + rxRelaxed := xurls.Relaxed() + genURL = rxRelaxed.FindString(results[0].Content()) + } + } + + if len(genURL) == 0 { + return "", errors.New("cannot find gift url") + } + //fmt.Printf("%#v\n", genURL) + resultURL, err := ExpandURL2(genURL, fbGameURL) + if err != nil { + return "", errors.New("cannot find gift url - ExpandURL2") + } + + return resultURL, nil +} + +// GetPostURL GetPostURL +func GetPostURL(htmlNode gokogirixml.Node) (string, error) { + + //s := QuerySelector(doc, "a._5pcq") + // + //url, ok := s.Attr("href") + //if !ok { + // //TODO: Fix and check all URLs + // rxRelaxed := xurls.Relaxed() + // url = rxRelaxed.FindString(doc.Text()) + // + // if len(url) == 0 { + // return "", errors.New("cannot find post url") + // } + //} else { + // url = "https://www.facebook.com" + url + //} + results, err := htmlNode.Search(xPathPostingURL) + if err != nil { + return "", err + } + + var genURL string + var attrib *gokogirixml.AttributeNode + if len(results) > 0 { + attrib = results[0].Attribute("href") + //fmt.Printf("%#v\n\n", attrib) + if attrib != nil { + tmpPostURL := attrib.Value() + //fmt.Printf("%#v\n", tmpPostURL) + tmpPostURL = html.UnescapeString(tmpPostURL) + //fmt.Printf("%#v\n", tmpPostURL) + tmpPostURL2, err := url.Parse(tmpPostURL) + //fmt.Printf("%#v -- %#v\n", tmpPostURL2, err) + if err != nil { + attrib = nil + goto nextPostLinkChecker + } + + //fmt.Printf("%#v\n", tmpPostURL2) + + newQuery := url.Values{} + oldQuery := tmpPostURL2.Query() + newQuery.Set("nx_source", oldQuery.Get("nx_source")) + newQuery.Set("gift_id", oldQuery.Get("gift_id")) + + tmpURLGen := url.URL{ + Scheme: tmpPostURL2.Scheme, + Host: tmpPostURL2.Host, + Path: `herowarsgame/posts/` + dataFT.TopLevelPostID, + } + if tmpURLGen.Scheme != "https" { + tmpURLGen.Scheme = "https" + } + if tmpURLGen.Host != "www.facebook.com" { + tmpURLGen.Host = "www.facebook.com" + } + + // dataFT. + + genURL = tmpURLGen.String() + } + //fmt.Printf("GENURL: %#v\n", genURL) + } +nextPostLinkChecker: + + resultURL, err := ExpandURL2(genURL, fbPageURL) + if err != nil { + return "", errors.New("cannot find post url - ExpandURL2") + } + + return resultURL, nil +} + +type object interface { + Find(string) *goquery.Selection +} + +// QuerySelector QuerySelector +func QuerySelector(s object, selector string) *goquery.Selection { + return s.Find(selector).First() +} + +// QuerySelectorEach QuerySelectorEach +func QuerySelectorEach(s object, selector string, mf func(i int, selection *goquery.Selection)) *goquery.Selection { + return s.Find(selector).Each(mf) +} + +// ParseTimeStamp ParseTimeStamp +func ParseTimeStamp(utime int64) (string, error) { + t := time.Unix(utime, 0) + return t.Format(time.RFC3339), nil +} + +// GetTimeStamp GetTimeStamp +func GetTimeStamp(htmlNode gokogirixml.Node) (string, error) { + //s := QuerySelector(doc, "._5ptz.timestamp.livetimestamp") + //s := QuerySelector(doc, "abbr._5ptz") + + results, err := htmlNode.Search(xPathArticle) + if err != nil { + return "", err + } + + if attrib := results[0].Attribute("data-ft"); attrib != nil { + json := attrib.Value() + normalJSON := html.UnescapeString(json) + dataFTinternal, err := UnmarshalDataFT([]byte(normalJSON)) + + dataFT = &dataFTinternal + + if err != nil { + + return "", err + } + pid := dataFTinternal.PageID + return ParseTimeStamp(dataFTinternal.PageInsights[pid].PostContext.PublishTime) + + } + + return "", errors.New("cannot find timestamp") +} + +// ProfileLink ProfileLink +type ProfileLink struct { + Name string + URL string +} + +// GetProfileLink GetProfileLink +func GetProfileLink(doc *goquery.Document) (*ProfileLink, error) { + s := QuerySelector(doc, "a.profileLink") + if s.Length() == 0 { + s = QuerySelector(doc, "span.fwb.fcg > a") + } + + pl := ProfileLink{} + + pl.Name = s.Text() + if pl.Name == "" { + return nil, errors.New("cannot find name of profile link") + } + + url, ok := s.Attr("href") + if !ok { + return nil, errors.New("cannot find url of profile link") + } + pl.URL = url + + return &pl, nil +} + +// GetBlogspotTimeStamp GetBlogspotTimeStamp +func GetBlogspotTimeStamp(doc *goquery.Document) (string, error) { + abbr := QuerySelector(doc, "a.timestamp-link > abbr") + t, ok := abbr.Attr("title") + if ok { + return t, nil + } + + return "", errors.New("cannot find timestamp") +} + +// GetBlogspotTitle GetBlogspotTitle +func GetBlogspotTitle(doc *goquery.Document) (string, error) { + t := QuerySelector(doc, "h3.post-title") + return strings.TrimSpace(t.Text()), nil +} + +//GetBlogspotContent GetBlogspotContent +func GetBlogspotContent(doc *goquery.Document) (string, error) { + c := QuerySelector(doc, "div.post-body") + + s, err := c.Html() + if err != nil { + return "", err + } + + var lines []string + + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + lines = append(lines, " "+scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return "", err + } + + return strings.Join(lines, "\n"), nil +} + +//GetBlogspotURL GetBlogspotURL +func GetBlogspotURL(doc *goquery.Document) (string, error) { + meta := QuerySelector(doc, "meta[property='og:url']") + u, ok := meta.Attr("content") + if ok { + return u, nil + } + + return "", errors.New("cannot find url") +} + +// GetBlogspotSummary GetBlogspotSummary +func GetBlogspotSummary(doc *goquery.Document) (string, error) { + meta := QuerySelector(doc, "meta[property='og:description']") + d, ok := meta.Attr("content") + if ok { + return strings.TrimSpace(d), nil + } + + return "", errors.New("cannot find summary") +} + +// GetBlogspotAuthor GetBlogspotAuthor +func GetBlogspotAuthor(doc *goquery.Document) (string, error) { + a := QuerySelector(doc, "span.post-author > span.fn") + return a.Text(), nil +} + +//GetBlogspotTags GetBlogspotTags +func GetBlogspotTags(doc *goquery.Document) (string, error) { + s := doc.Find("span.post-labels > a") + labels := "" + s.Each(func(_ int, l *goquery.Selection) { + if labels != "" { + labels += ", " + } + labels += l.Text() + }) + return labels, nil +} + +// ParseBlogspotPost ParseBlogspotPost +func ParseBlogspotPost(doc *goquery.Document) (*FBPostData, error) { + bs := FBPostData{} + var err error + + bs.TimeStamp, err = GetBlogspotTimeStamp(doc) + if err != nil { + return &bs, err + } + + bs.Title, err = GetBlogspotTitle(doc) + if err != nil { + return &bs, err + } + + bs.Content, err = GetBlogspotContent(doc) + if err != nil { + return &bs, err + } + + bs.PostURL, err = GetBlogspotURL(doc) + if err != nil { + return &bs, err + } + + bs.Summary, err = GetBlogspotSummary(doc) + if err != nil { + return &bs, err + } + + bs.Author, err = GetBlogspotAuthor(doc) + if err != nil { + return &bs, err + } + + bs.Tags, err = GetBlogspotTags(doc) + if err != nil { + return &bs, err + } + + return &bs, nil +} diff --git a/fb_dataft.go b/fb_dataft.go new file mode 100644 index 0000000..52250d4 --- /dev/null +++ b/fb_dataft.go @@ -0,0 +1,64 @@ +// This file was generated from JSON Schema using quicktype, do not modify it directly. +// To parse and unparse this JSON data, add this code to your project and do: +// +// dataFT, err := UnmarshalDataFT(bytes) +// bytes, err = dataFT.Marshal() + +package main + +import "encoding/json" + +func UnmarshalDataFT(data []byte) (DataFT, error) { + var r DataFT + err := json.Unmarshal(data, &r) + return r, err +} + +func (r *DataFT) Marshal() ([]byte, error) { + return json.Marshal(r) +} + +type DataFT struct { + MFStoryKey string `json:"mf_story_key"` + TopLevelPostID string `json:"top_level_post_id"` + TlObjid string `json:"tl_objid"` + ContentOwnerIDNew string `json:"content_owner_id_new"` + ThrowbackStoryFbid string `json:"throwback_story_fbid"` + PageID string `json:"page_id"` + PhotoID string `json:"photo_id"` + StoryLocation int64 `json:"story_location"` + StoryAttachmentStyle string `json:"story_attachment_style"` + PageInsights map[string]PageInsight `json:"page_insights"` + Tn string `json:"tn"` +} + +type PageInsight struct { + PageID string `json:"page_id"` + ActorID string `json:"actor_id"` + Dm Dm `json:"dm"` + Psn string `json:"psn"` + PostContext PostContext `json:"post_context"` + Role int64 `json:"role"` + Sl int64 `json:"sl"` + Targets []Target `json:"targets"` +} + +type Dm struct { + IsShare int64 `json:"isShare"` + OriginalPostOwnerID int64 `json:"originalPostOwnerID"` +} + +type PostContext struct { + ObjectFbtype int64 `json:"object_fbtype"` + PublishTime int64 `json:"publish_time"` + StoryName string `json:"story_name"` + StoryFbid []string `json:"story_fbid"` +} + +type Target struct { + ActorID string `json:"actor_id"` + PageID string `json:"page_id"` + PostID string `json:"post_id"` + Role int64 `json:"role"` + ShareID int64 `json:"share_id"` +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ea1770c --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module git.deineagentur.com/Jomaar/fbBot + +go 1.13 + +require ( + github.com/PuerkitoBio/goquery v1.5.0 + github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 + github.com/chromedp/chromedp v0.5.2 + github.com/fsnotify/fsnotify v1.4.7 + github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f + github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546 + github.com/mailru/easyjson v0.7.0 + github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756 + github.com/microcosm-cc/bluemonday v1.0.2 + github.com/robfig/cron v1.2.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/spf13/viper v1.6.1 + gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc + mvdan.cc/xurls/v2 v2.1.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c8ac99d --- /dev/null +++ b/go.sum @@ -0,0 +1,179 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 h1:QD3KxSJ59L2lxG6MXBjNHxiQO2RmxTQ3XcK+wO44WOg= +github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/chromedp v0.5.2 h1:W8xBXQuUnd2dZK0SN/lyVwsQM7KgW+kY5HGnntms194= +github.com/chromedp/chromedp v0.5.2/go.mod h1:rsTo/xRo23KZZwFmWk2Ui79rBaVRRATCjLzNQlOFSiA= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f h1:6UIvzqlGM38lOpKP380Wbl0kUyyjutcc7KJUaDM/U4o= +github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f/go.mod h1:C3R3VzPq+DAwilxue7DiV6F2QL1rrQX0L56GyI+sBxM= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546 h1:hqxaQP14eTbeZGHZhsDInzj9sJAnEufjVQL4bEA/p+8= +github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546/go.mod h1:lUUaVYlpAQ1Oo6vIZfec6CXQZjOvFZLyqaR8Dl7m+hk= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756 h1:m3Cy2iq0HKujZ4uN59r3XrEZ0ijwcvCOXxgT2pXJEGU= +github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756/go.mod h1:e8zJNnMchIy3s0Lp9XanSSHmYnGWlbdmBUG81EhrNRU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= +github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc h1:LMEBgNcZUqXaP7evD1PZcL6EcDVa2QOFuI+cqM3+AJM= +gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc/go.mod h1:N8UOSI6/c2yOpa/XDz3KVUiegocTziPiqNkeNTMiG1k= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +mvdan.cc/xurls v1.1.0 h1:kj0j2lonKseISJCiq1Tfk+iTv65dDGCl0rTbanXJGGc= +mvdan.cc/xurls/v2 v2.1.0 h1:KaMb5GLhlcSX+e+qhbRJODnUUBvlw01jt4yrjFIHAuA= +mvdan.cc/xurls/v2 v2.1.0/go.mod h1:5GrSd9rOnKOpZaji1OZLYL/yeAAtGDlo/cFe+8K5n8E= diff --git a/main.go b/main.go new file mode 100644 index 0000000..c39d785 --- /dev/null +++ b/main.go @@ -0,0 +1,203 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/chromedp/chromedp" + "github.com/fsnotify/fsnotify" + "github.com/lunny/html2md" + "github.com/robfig/cron/v3" + "github.com/spf13/viper" +) + +var ( + // GlobalApp holds a pointer for the App + cronJob *cron.Cron +) + +func init() { + viper.SetDefault("development_mode", true) + viper.SetDefault("cronjob_interval", "@every 5m") + viper.SetConfigType("yaml") + viper.SetConfigName("config") // name of config file (without extension) + viper.AddConfigPath("/etc/fbBot/") // path to look for the config file in + viper.AddConfigPath("$HOME/.fbBot") // call multiple times to add many search paths + viper.AddConfigPath(".") // optionally look for config in the working directory + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + // any approach to require this configuration into your program. + var yamlExample = []byte(` +development_mode: true +cronjob_interval: "@every 5m" +no_run_on_start: false +http_useragent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 + (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19 +webhooks: + live: + - https://discordapp.com/api/webhooks/... + - https://discordapp.com/api/webhooks/... + devel: + - https://discordapp.com/api/webhooks/... +comment_on_posts: +- TY +- ty! +- danke! +- I could use it! +- thx! +- Thank you! +- Merci +- Yeah! +- Vielen Dank! +login: + email: example@example.com + passwd: thefbpassword +`) + + viper.ReadConfig(bytes.NewBuffer(yamlExample)) + viper.WriteConfig() + } else { + // Config file was found but another error was produced + } + } + viper.WatchConfig() + viper.OnConfigChange(func(e fsnotify.Event) { + fmt.Println("Config file changed:", e.Name) + }) + + html2md.AddRule("span", &html2md.Rule{ + Patterns: []string{"span"}, + Tp: html2md.Void, + Replacement: func(innerHTML string, attrs []string) string { + return innerHTML + }, + }) + +} + +func main() { + SetupCloseHandler() + if viper.GetBool("no_run_on_start") { + go cronTask() + } + + cronJob = cron.New() + cronJob.AddFunc(viper.GetString("cronjob_interval"), func() { + cronTask() + }) + cronJob.Start() + + for { + time.Sleep(30 * time.Second) + } +} + +func postRequest(content *Webhook) { + + webhooks := viper.GetStringSlice("webhooks.live") + if viper.GetBool("development_mode") { + webhooks = viper.GetStringSlice("webhooks.devel") + } + + jsonStrBytes, err := content.Marshal() + if err != nil { + log.Println("postRequest error:", err.Error()) + return + } + + for _, webhook := range webhooks { + //fmt.Println("URL:>", webhook) + + req, err := http.NewRequest("POST", webhook, bytes.NewBuffer(jsonStrBytes)) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Println("ERROR: " + err.Error()) + } + defer resp.Body.Close() + + log.Println("response Status:", resp.Status) + //fmt.Println("response Headers:", resp.Header) + } +} + +func cronTask() error { + dir := filepath.Join(".", "content") + os.MkdirAll(dir, os.ModePerm) + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.DisableGPU, + chromedp.UserAgent(viper.GetString("http_useragent")), + chromedp.UserDataDir(dir), + ) + + allocCtx, cancelAll := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancelAll() + + // also set up a custom logger + taskCtx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf), chromedp.WithErrorf(log.Printf)) + defer cancel() + + chromedp.Run(taskCtx) + + page1(taskCtx) + page2(taskCtx) + page3(taskCtx) + page4(taskCtx) + page5(taskCtx) + page6(taskCtx) + page7(taskCtx) + + postID := "" + var fbPost *FBPostData + var fbPostErr error + postID, fbPost, fbPostErr = page8(taskCtx) + fmt.Printf("Error: %#v\n", fbPostErr) + if fbPost.TimeStamp != lastUpdatePosted { + page9(taskCtx, postID) + } + + time.Sleep(2 * time.Second) + + if fbPost.TimeStamp != lastUpdatePosted { + timeStamp := fbPost.TimeStamp + pTime, err := time.Parse(time.RFC3339, fbPost.TimeStamp) + if err == nil { + timeStamp = pTime.UTC().Format("2006-01-02 15:04:05 UTC") + } + embeded := []*Embed{} + + embeded = append(embeded, &Embed{ + Title: fbPost.Title, + Description: fbPost.Content + ` + + Date of Post: ` + timeStamp + ` + Post URL: ` + fbPost.PostURL, + Image: Image{ + URL: fbPost.ImageURL, + }, + URL: fbPost.GiftURL, + }) + + dcontent := &Webhook{ + Content: fbPost.Title, + Embeds: embeded, + } + if fbPostErr == nil { + postRequest(dcontent) + } + lastUpdatePosted = fbPost.TimeStamp + viper.Set("lastPostTimestamp", fbPost.TimeStamp) + viper.WriteConfig() + } + return nil + +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..f9acccc --- /dev/null +++ b/main_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "html" + "io/ioutil" + "net/url" + "strings" + "testing" +) + +var ( + testStyleURL string = `https\3a //scontent.fbre2-1.fna.fbcdn.net/v/t1.0-9/cp0/e15/q65/s720x720/82084740_2291078311193333_4974989532400189440_o.jpg?_nc_cat\3d 1\26 efg\3d eyJpIjoidCJ9\26 _nc_ohc\3d YFSkDqncwkYAX9wymCm\26 _nc_ht\3d scontent.fbre2-1.fna\26 oh\3d a31ebd3e26518967fa5222b85098d300\26 oe\3d 5EAF6A0D` + testMoopURL string = `/story.php?story_fbid=2291080917859739&id=1574763202824851&_ft_=mf_story_key.2291080917859739%3Atop_level_post_id.2291080917859739%3Atl_objid.2291080917859739%3Acontent_owner_id_new.1574763202824851%3Athrowback_story_fbid.2291080917859739%3Apage_id.1574763202824851%3Aphoto_id.2291078307860000%3Astory_location.4%3Astory_attachment_style.photo%3Apage_insights.%7B%221574763202824851%22%3A%7B%22page_id%22%3A1574763202824851%2C%22actor_id%22%3A1574763202824851%2C%22dm%22%3A%7B%22isShare%22%3A0%2C%22originalPostOwnerID%22%3A0%7D%2C%22psn%22%3A%22EntStatusCreationStory%22%2C%22post_context%22%3A%7B%22object_fbtype%22%3A266%2C%22publish_time%22%3A1578841213%2C%22story_name%22%3A%22EntStatusCreationStory%22%2C%22story_fbid%22%3A%5B2291080917859739%5D%7D%2C%22role%22%3A1%2C%22sl%22%3A4%2C%22targets%22%3A%5B%7B%22actor_id%22%3A1574763202824851%2C%22page_id%22%3A1574763202824851%2C%22post_id%22%3A2291080917859739%2C%22role%22%3A1%2C%22share_id%22%3A0%7D%5D%7D%7D&__tn__=-R` +) + +func Test_main(t *testing.T) { + genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`) + genURL = strings.ReplaceAll(genURL, `\3d `, `=`) + genURL = strings.ReplaceAll(genURL, `\26 `, `&`) + fmt.Printf("%#v\n", genURL) +} + +func TestCronTask(t *testing.T) { + fmt.Printf("%#v", cronTask()) +} + +func TestStartTask(t *testing.T) { + fmt.Printf("%#v", startTask()) +} + +func Test_moop(t *testing.T) { + + genURL, err := url.Parse(html.UnescapeString(testMoopURL)) + if err != nil { + t.Errorf("%#v", err.Error()) + } + + fmt.Printf("%#v\n", genURL) + for i, query := range genURL.Query() { + fmt.Printf("%s : %#v\n", i, query) + if i == "_ft_" { + text := "" + for _, txt := range query { + text = text + txt + } + for in, mtext := range strings.SplitN(text, ":", 10) { + allText := strings.Split(mtext, ".") + fmt.Printf("%d ::: %s : %s\n", in, allText[0], allText[1]) + } + } + } +} + +func TestParse(t *testing.T) { + url := "https://www.facebook.com/herowarsgame/posts/" + //url := "https://www.facebook.com/jayasaro.panyaprateep.org/posts/1095007907274561:0" + //url := "https://www.facebook.com/jayasaro.panyaprateep.org/photos/a.318290164946343.68815.318196051622421/1119561951485823/?type=3" + //result, err := Parse(url) + dat, err := ioutil.ReadFile("./2020-01-14T17:07:58+01:00-PAGE8-content.html") + if err != nil { + t.Error(err) + return + } + //fmt.Print(string(dat)) + + post, err := ParsePost(string(dat), url) + if err != nil { + t.Error(err) + return + } + t.Log(post) + fmt.Printf("%#v", post) + +} diff --git a/page_call.go b/page_call.go new file mode 100644 index 0000000..cf492f5 --- /dev/null +++ b/page_call.go @@ -0,0 +1,272 @@ +package main + +import ( + "context" + "encoding/json" + "log" + "math/rand" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/cdproto/runtime" + "github.com/chromedp/chromedp" + "github.com/spf13/viper" +) + +func page1(taskCtx context.Context) { + page := "PAGE1" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + // ensure that the browser process is started + if err := chromedp.Run(c, + network.Enable(), + network.SetExtraHTTPHeaders(network.Headers(headers)), + chromedp.Navigate("https://m.facebook.com/"), + //chromedp.Nodes(`document`, &nodes, chromedp.ByJSPath), + chromedp.WaitVisible(emailField), + chromedp.WaitVisible(passwordField), + chromedp.SendKeys(emailField, viper.GetString("login.email")), + chromedp.SendKeys(passwordField, viper.GetString("login.passwd")), + chromedp.WaitReady(`//*[@id="signup-button"]`), + chromedp.Click(`//*[@name="login"]`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) +} + +func page2(taskCtx context.Context) { + return // currently not need + page := "PAGE2" + + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + // ensure that the browser process is started + if err := chromedp.Run(c, + network.Enable(), + network.SetExtraHTTPHeaders(network.Headers(headers)), + chromedp.Navigate("https://m.facebook.com/"), + //chromedp.Nodes(`document`, &nodes, chromedp.ByJSPath), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) +} + +func page3(taskCtx context.Context) { + page := "PAGE3" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + if err := chromedp.Run(c, + chromedp.WaitVisible(`div._5xu4 > form`), + chromedp.Submit(`div._5xu4 > form`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) +} + +func page4(taskCtx context.Context) { + page := "PAGE4" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + if err := chromedp.Run(c, + chromedp.WaitVisible(`#u_0_0`), + chromedp.SendKeys(`#u_0_0`, viper.GetString("login.passwd")), + chromedp.WaitVisible(`div._4g34 > form`), + chromedp.Submit(`div._4g34 > form`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) + +} + +func page5(taskCtx context.Context) { + page := "PAGE5" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + if err := chromedp.Run(c, + chromedp.WaitVisible(`div._4g34 > div._2pii > div > a`), + chromedp.Click(`div._4g34 > div._2pii > div > a`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShotOnly(taskCtx, page) +} + +func page6(taskCtx context.Context) { + page := "PAGE6" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + // ensure that the browser process is started + if err := chromedp.Run(c, + network.Enable(), + network.SetExtraHTTPHeaders(network.Headers(headers)), + chromedp.Navigate("https://m.facebook.com/herowarsgame/posts/"), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) +} + +func page7(taskCtx context.Context) { + page := "PAGE7" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + defer cancelCtxWTO() + + // ensure that the browser process is started + if err := chromedp.Run(c, + chromedp.WaitNotPresent(`a._15ko._77li.touchable._77la`), + chromedp.WaitVisible(`a._15ko._77li.touchable`), + chromedp.Click(`a._15ko._77li.touchable`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + + makeScreenShot(taskCtx, page) +} + +func page8(taskCtx context.Context) (postID string, fbPost *FBPostData, err error) { + page := "PAGE8" + fbPost = nil + err = nil + + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) + defer cancelCtxWTO() + + const function1 = `(function(d) { + var b = d.evaluate('//*/div[@id="pages_msite_body_contents"]/div/div[3]/div[1]/div/article', document, null, XPathResult.ANY_TYPE, null ); + let bb = b.iterateNext(); + let json = JSON.parse(bb.dataset.ft) + console.log(json) + return json.top_level_post_id + })(document);` + + // ensure that the browser process is started + if err := chromedp.Run(c, + chromedp.WaitVisible(`a._15ko._77li.touchable`), + chromedp.ActionFunc(func(ctx context.Context) error { + tmpPostID, exp, err := runtime.Evaluate(function1).Do(ctx) + if err != nil { + log.Println(err) + return err + } + if exp != nil { + log.Println(exp) + return exp + } + + err = json.Unmarshal(tmpPostID.Value, &postID) + if err != nil { + log.Println(err) + return err + } + + return nil + }), + chromedp.Sleep(1*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return "", nil, err + //panic(err) + } + + fbPost, err = makeScreenShotAndParsePost(taskCtx, page) + + return postID, fbPost, err +} + +func page9(taskCtx context.Context, postID string) { + page := "PAGE9" + c, cancelCtxWTO := context.WithTimeout(taskCtx, 15*time.Second) + defer cancelCtxWTO() + + clickSel := `//*[@id="composer-` + postID + `"]/form/div[1]/div[3]/button` + //formSel := `//*[@id="composer-` + postID + `"]/form` + + if postID != "" { + // ensure that the browser process is started + if err := chromedp.Run(c, + chromedp.WaitVisible(`//*[@id="feedback_inline_`+postID+`"]/div[2]/div[2]/a`), + chromedp.Click(`//*[@id="feedback_inline_`+postID+`"]/div[2]/div[2]/a`), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + return + //panic(err) + } + } + + makeScreenShot(taskCtx, page) + + page = page + "-1" + // ensure that the browser process is started + if err := chromedp.Run(c, + chromedp.WaitVisible(`//*/textarea[@id="composerInput"]`), + chromedp.SendKeys(`//*/textarea[@id="composerInput"]`, sendRandomText()), + chromedp.Sleep(time.Second), + chromedp.WaitVisible(clickSel), + chromedp.WaitEnabled(clickSel), + chromedp.Click(clickSel), + //chromedp.Sleep(2*time.Second), + chromedp.Sleep(2*time.Second), + ); err != nil { + log.Println(page) + log.Println(err) + panic(err) + //return + + } + + makeScreenShot(taskCtx, page) +} + +func sendRandomText() string { + s := rand.NewSource(time.Now().Unix()) + r := rand.New(s) // initialize local pseudorandom generator + return randomText[r.Intn(len(randomText))] + +} diff --git a/screenshot.go b/screenshot.go new file mode 100644 index 0000000..16c6885 --- /dev/null +++ b/screenshot.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "io/ioutil" + "math" + "time" + + "github.com/spf13/viper" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" +) + +func makeScreenShot(c context.Context, page string) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + var content string + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + //log.Fatal(err) + } + } + +} + +func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, error) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + var content string + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + //log.Fatal(err) + } + } + return ParsePost(content, "https://www.facebook.com/pg/herowarsgame/posts/?ref=page_internal") + +} + +func makeScreenShotOnly(c context.Context, page string) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + } + +} + +// fullScreenshot takes a screenshot of the entire browser viewport. +// +// Liberally copied from puppeteer's source. +// +// Note: this will override the viewport emulation settings. +func fullScreenshot(quality int64, res *[]byte, content *string) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Sleep(1 * time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + // get layout metrics + _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + // force viewport emulation + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + // capture screenshot + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + chromedp.OuterHTML("html", content), + } +} + +// fullScreenshotOnly takes a screenshot of the entire browser viewport. +// +// Liberally copied from puppeteer's source. +// +// Note: this will override the viewport emulation settings. +func fullScreenshotOnly(quality int64, res *[]byte) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Sleep(2 * time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + // get layout metrics + _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + // force viewport emulation + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + // capture screenshot + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + } +} diff --git a/urlexpander.go b/urlexpander.go new file mode 100644 index 0000000..19e0ab3 --- /dev/null +++ b/urlexpander.go @@ -0,0 +1,66 @@ +package main + +import ( + "net/http" + "strings" +) + +// ExpandURL ExpandURL +func ExpandURL(url string) (string, error) { + expandedURL := url + + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + expandedURL = req.URL.String() + return nil + }, + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + return expandedURL, nil +} + +func ExpandURL2(myURL, searchURL string) (string, error) { + nextURL := myURL + var i int + for i < 100 { + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }} + + resp, err := client.Get(nextURL) + + if err != nil { + return "", err + } + + //fmt.Println("StatusCode:", resp.StatusCode) + //fmt.Println(resp.Request.URL) + + if resp.StatusCode == 200 { + //fmt.Println("Done!") + break + } else { + nextURLTest := resp.Header.Get("Location") + if strings.Contains(nextURL, searchURL) { + break + } + nextURL = nextURLTest + i++ + } + } + + return nextURL, nil +} diff --git a/webhook.go b/webhook.go new file mode 100644 index 0000000..1116ca2 --- /dev/null +++ b/webhook.go @@ -0,0 +1,54 @@ +package main + +import "encoding/json" + +func UnmarshalWebhook(data []byte) (Webhook, error) { + var r Webhook + err := json.Unmarshal(data, &r) + return r, err +} + +func (r *Webhook) Marshal() ([]byte, error) { + return json.Marshal(r) +} + +type Webhook struct { + Username string `json:"username,omitempty"` + AvatarURL string `json:"avatar_url,omitempty"` + TTS string `json:"tts,omitempty"` + Content string `json:"content,omitempty"` + Embeds []*Embed `json:"embeds,omitempty"` +} + +type Embed struct { + Author Author `json:"author,omitempty"` + Title string `json:"title,omitempty"` + URL string `json:"url,omitempty"` + Description string `json:"description,omitempty"` + Color int64 `json:"color,omitempty"` + Fields []Field `json:"fields,omitempty"` + Thumbnail Image `json:"thumbnail,omitempty"` + Image Image `json:"image,omitempty"` + Footer Footer `json:"footer,omitempty"` +} + +type Author struct { + Name string `json:"name"` + URL string `json:"url"` + IconURL string `json:"icon_url"` +} + +type Field struct { + Name string `json:"name"` + Value string `json:"value"` + Inline *bool `json:"inline,omitempty"` +} + +type Footer struct { + Text string `json:"text"` + IconURL string `json:"icon_url"` +} + +type Image struct { + URL string `json:"url"` +}