HeroWarsTools/hw-fbbot
1
0
Fork 0
Code Releases Activity

first commit

Browse Source
This commit is contained in:
Josef Fröhle
2020-01-15 13:35:30 +01:00
commit c9cd268366
15 changed files with 1884 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
content/

33
README.md Normal file
View File

@@ -0,0 +1,33 @@
# Dependencies
- libxml2-dev libonig-dev
- chrome / chromium browser
# Config
The Config File `config.yaml` should be located in /etc/fbbot.
```yaml
development_mode: true
cronjob_interval: "@every 5m"
no_run_on_start: false
http_useragent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19
(KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19
webhooks:
live:
- https...
- https
devel:
- https...
- https
comment_on_posts:
- TY
- ty!
- danke!
- I could use it!
- thx!
- Thank you!
- Merci
- Yeah!
- Vielen Dank!
```

21
closehandle.go Normal file
View File

@@ -0,0 +1,21 @@
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
)
// SetupCloseHandler creates a 'listener' on a new goroutine which will notify the
// program if it receives an interrupt from the OS. We then handle this by calling
// our clean up procedure and exiting the program.
func SetupCloseHandler() {
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
fmt.Println("\r- Ctrl+C pressed in Terminal")
os.Exit(0)
}()
}

20
configuration.go Normal file
View File

@@ -0,0 +1,20 @@
package main
import (
"github.com/chromedp/cdproto/cdp"
"github.com/spf13/viper"
)
const (
emailField = `//*[@id="m_login_email"]`
passwordField = `//*[@id="m_login_password"]`
)
var (
headers = map[string]interface{}{
"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
}
nodes []*cdp.Node
randomText = viper.GetStringSlice("comment_on_posts")
)

10
content_functions.go Normal file
View File

@@ -0,0 +1,10 @@
package main
import "strings"
func getImageURL(testStyleURL string) (string, error) {
genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`)
genURL = strings.ReplaceAll(genURL, `\3d `, `=`)
genURL = strings.ReplaceAll(genURL, `\26 `, `&`)
return genURL, nil
}

691
facebook.go Normal file
View File

@@ -0,0 +1,691 @@
package main
import (
"bufio"
"errors"
"html"
"net/url"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/jbowtie/gokogiri"
gokogirixml "github.com/jbowtie/gokogiri/xml"
"github.com/lunny/html2md"
"github.com/microcosm-cc/bluemonday"
"mvdan.cc/xurls/v2"
)
// xPath constants
const (
xPathArticle string = `(//*/article[contains(concat(" ", normalize-space(@class)," "), " _55wo ") and contains(concat(" ", normalize-space(@class)," "), " _5rgr ")])[1]`
xPathArticleContent string = xPathArticle + `/div/div[contains(concat(" ", normalize-space(@class)," "), " _5rgt ") and contains(concat(" ", normalize-space(@class)," "), " _5nk5 ")]/span`
xPathImageURL string = xPathArticle + `/div/div[2]/div[1]/a/div/div/i`
xPathImageURL2 string = xPathArticle + `/div/div[2]/section/div/i`
xPathImageURL3 string = xPathArticle + `/div/div[2]/section/div/div/i` // Video Preview
xPathGiftURL string = xPathArticle + `/div/div[2]/section/a`
xPathPostingURL string = xPathArticle + `/div/header/div[2]/div/div/div[1]/div/a`
)
// internal URL constants
const (
fbPageURL string = "https://www.facebook.com/herowarsgame/"
webhookLive string = "https://discordapp.com/api/webhooks/..."
webhookLiveExcelsior string = "https://discordapp.com/api/webhooks/..."
webhookDev string = "https://discordapp.com/api/webhooks/..."
// fbGameURL for search on expaned urls
fbGameURL string = "apps.facebook.com/mobaheroes"
)
var (
dataFT *DataFT
lastUpdatePosted string
regexStyleImage = regexp.MustCompile(`(?m)url\((.*)\);`)
)
// Constants for all search for title of the Postings
const (
FreeTitanArtifact string = "FREE Titan Artifact"
FreeSilverCaskets string = "FREE Silver Caskets"
FreeSoulStones string = "Soul Stones"
FreeSkinStones string = "Skin Stones"
ActionKeepTheAmount string = "Keep the amount"
FreeWinterfestBaubles string = "Winterfest Baubles"
FreeTopFanPackage string = "Top Fan"
FreeEnergyForFee string = "ENERGY FOR FREE"
WinterfestRankingRewards string = "Winterfest ranking rewards"
)
//FBPostData FBPostData
type FBPostData struct {
PostURL string
TimeStamp string
ProfileLink *ProfileLink
GiftURL string
ImageURL string
Content string
Summary string
Title string
Author string
Tags string
}
// ParsePost ParsePost
func ParsePost(s, PostURL string) (*FBPostData, error) {
fb := FBPostData{PostURL: PostURL}
docKogiri, err := gokogiri.ParseHtml([]byte(s))
if err != nil {
return &fb, err
}
defer docKogiri.Free()
htmlNode := docKogiri.Root().FirstChild()
//doc, err := goquery.NewDocumentFromReader(strings.NewReader(s))
//if err != nil {
// return &fb, err
//}
fb.TimeStamp, err = GetTimeStamp(htmlNode)
if err != nil {
return &fb, err
}
//fmt.Printf("\n\n%#v\n\n", dataFT)
//fb.ProfileLink, err = GetProfileLink(doc)
//if err != nil {
// return &fb, err
//}
fb.ImageURL, err = GetImageURL(htmlNode)
if err != nil {
return &fb, err
}
fb.GiftURL, err = GetGiftURL(htmlNode)
if err != nil {
return &fb, err
}
fb.PostURL, err = GetPostURL(htmlNode)
if err != nil {
fb.PostURL = PostURL
return &fb, err
}
fb.Content, err = GetContent(htmlNode)
if err != nil {
return &fb, err
}
if strings.Contains(fb.Content, FreeSilverCaskets) {
if len(fb.Title) == 0 {
fb.Title = FreeSilverCaskets
} else {
fb.Title = FreeSilverCaskets + "+" + fb.Title
}
}
if strings.Contains(fb.Content, FreeTitanArtifact) {
if len(fb.Title) == 0 {
fb.Title = FreeTitanArtifact
} else {
fb.Title = FreeTitanArtifact + "+" + fb.Title
}
}
if strings.Contains(fb.Content, FreeSoulStones) {
if len(fb.Title) == 0 {
fb.Title = FreeSoulStones
} else {
fb.Title = FreeSoulStones + "+" + fb.Title
}
}
if strings.Contains(fb.Content, FreeSkinStones) {
if len(fb.Title) == 0 {
fb.Title = FreeSkinStones
} else {
fb.Title = FreeSkinStones + "+" + fb.Title
}
}
if strings.Contains(fb.Content, ActionKeepTheAmount) {
if len(fb.Title) == 0 {
fb.Title = ActionKeepTheAmount
} else {
fb.Title = ActionKeepTheAmount + "+" + fb.Title
}
fb.GiftURL = fb.PostURL
}
if strings.Contains(fb.Content, FreeWinterfestBaubles) {
if len(fb.Title) == 0 {
fb.Title = FreeWinterfestBaubles
} else {
fb.Title = FreeWinterfestBaubles + "+" + fb.Title
}
}
if strings.Contains(fb.Content, FreeTopFanPackage) {
if len(fb.Title) == 0 {
fb.Title = FreeTopFanPackage
} else {
fb.Title = FreeTopFanPackage + "+" + fb.Title
}
}
if strings.Contains(fb.Content, FreeEnergyForFee) {
if len(fb.Title) == 0 {
fb.Title = FreeEnergyForFee
} else {
fb.Title = FreeEnergyForFee + "+" + fb.Title
}
}
if strings.Contains(fb.Content, WinterfestRankingRewards) {
if len(fb.Title) == 0 {
fb.Title = WinterfestRankingRewards
} else {
fb.Title = WinterfestRankingRewards + "+" + fb.Title
}
}
if len(fb.Title) == 0 {
fb.Title = "unknown - need to implemented"
}
defer func() {
dataFT = nil
}()
return &fb, nil
}
// Parse Parse
func Parse(url string) (*FBPostData, error) {
doc, err := goquery.NewDocument(url)
if err != nil {
return nil, err
}
if strings.Contains(url, ".blogspot.") {
return ParseBlogspotPost(doc)
}
// If not login, post looks like
// <div class="hidden_elem"><code id="u_0_p"><!-- ... --></code></div>
s := QuerySelector(doc, "div.hidden_elem > code")
cmt, err := s.Html()
if err != nil {
return nil, err
}
if len(cmt) != 0 {
return ParsePost(cmt, url)
}
s = QuerySelector(doc, "div._427x")
cmt, err = s.Html()
if err != nil {
return nil, err
}
return ParsePost(cmt, url)
}
// ParseAll ParseAll
func ParseAll(url string) ([]*FBPostData, error) {
doc, err := goquery.NewDocument(url)
if err != nil {
return nil, err
}
allFbPosts := []*FBPostData{}
QuerySelectorEach(doc, "div._427x", func(i int, selected *goquery.Selection) {
cmt, err := selected.Html()
if err != nil {
return
}
fbPost, err := ParsePost(cmt, url)
if err != nil {
return
}
allFbPosts = append(allFbPosts, fbPost)
})
return allFbPosts, nil
}
// GetContent GetContent
func GetContent(htmlNode gokogirixml.Node) (string, error) {
results, err := htmlNode.Search(xPathArticleContent)
if err != nil {
//fmt.Printf("ERR: %#v -- %#v\n", results, err)
return "", err
}
//fmt.Printf("ERR: %#v -- %#v\n", results, err)
if len(results) > 0 {
if resultHTML := results[0].InnerHtml(); resultHTML != "" {
bmsanizer := bluemonday.StrictPolicy()
bmsanizer.AllowAttrs("href").OnElements("a")
bmsanizer.AllowElements("p")
bmsanizer.RequireParseableURLs(true)
htmlText := bmsanizer.SanitizeBytes([]byte(resultHTML))
content := strings.Join(strings.Fields(string(htmlText)), " ")
content = html2md.Convert(content)
content = html.UnescapeString(content)
content = strings.TrimSpace(content)
return content, nil
}
}
return "", errors.New("no content found")
}
// GetImageURL GetImageURL
func GetImageURL(htmlNode gokogirixml.Node) (string, error) {
results, err := htmlNode.Search(xPathImageURL)
if err != nil {
//fmt.Printf("ERR: %#v -- %#v\n", results, err)
return "", err
}
//fmt.Printf("RESULT: %#v -- %#v\n", results, err)
// search for the second possible image
if len(results) == 0 {
results, err = htmlNode.Search(xPathImageURL2)
if err != nil {
//fmt.Printf("ERR2: %#v -- %#v\n", results, err)
return "", err
}
}
// Video Preview
if len(results) == 0 {
results, err = htmlNode.Search(xPathImageURL3)
if err != nil {
//fmt.Printf("ERR2: %#v -- %#v\n", results, err)
return "", err
}
}
//fmt.Printf("RESULT2: %#v -- %#v\n", results, err)
if len(results) > 0 {
if attrib := results[0].Attribute("style"); attrib != nil {
styleContent := attrib.Value()
codedImageURL := regexStyleImage.FindString(styleContent)
genURL := strings.ReplaceAll(codedImageURL, `\3a `, `:`)
genURL = strings.ReplaceAll(genURL, `\3d `, `=`)
genURL = strings.ReplaceAll(genURL, `\26 `, `&`)
genURL = strings.ReplaceAll(genURL, `\25 `, `%`)
genURL = strings.TrimPrefix(genURL, `url('`)
genURL = strings.TrimSuffix(genURL, `');`)
return genURL, nil
}
}
return "", errors.New("cannot find image url")
//s := QuerySelector(doc, "img.scaledImageFitHeight")
//if s.Length() == 0 {
// s = QuerySelector(doc, "img.scaledImageFitWidth")
//}
//
//url, ok := s.Attr("src")
//if !ok {
// return "", errors.New("cannot find image url")
//}
//
//return url, nil
}
// GetGiftURL GetGiftURL
func GetGiftURL(htmlNode gokogirixml.Node) (string, error) {
//s := QuerySelector(doc, "div._6ks > a")
//
//url, ok := s.Attr("href")
results, err := htmlNode.Search(xPathGiftURL)
if err != nil {
// search for the second possible image
results, err = htmlNode.Search(xPathImageURL2)
if err != nil {
return "", err
}
}
var genURL string
var attrib *gokogirixml.AttributeNode
if len(results) > 0 {
attrib = results[0].Attribute("href")
if attrib != nil {
tmpGiftURL := attrib.Value()
tmpGiftURL = html.UnescapeString(tmpGiftURL)
tmpGiftURL2, err := url.Parse(tmpGiftURL)
if err != nil {
attrib = nil
goto nextGiftLinkChecker
}
//fmt.Printf("%#v\n", tmpGiftURL2)
newQuery := url.Values{}
oldQuery := tmpGiftURL2.Query()
newQuery.Set("nx_source", oldQuery.Get("nx_source"))
newQuery.Set("gift_id", oldQuery.Get("gift_id"))
tmpURLGen := url.URL{
Scheme: tmpGiftURL2.Scheme,
Host: tmpGiftURL2.Host,
Path: tmpGiftURL2.Path,
RawQuery: newQuery.Encode(),
}
if tmpURLGen.Scheme != "https" {
tmpURLGen.Scheme = "https"
}
genURL = tmpURLGen.String()
}
}
nextGiftLinkChecker:
if attrib == nil {
results, err := htmlNode.Search(xPathArticleContent)
if err != nil {
return "", err
}
if len(results) > 0 {
//TODO: Fix and check all URLs
rxRelaxed := xurls.Relaxed()
genURL = rxRelaxed.FindString(results[0].Content())
}
}
if len(genURL) == 0 {
return "", errors.New("cannot find gift url")
}
//fmt.Printf("%#v\n", genURL)
resultURL, err := ExpandURL2(genURL, fbGameURL)
if err != nil {
return "", errors.New("cannot find gift url - ExpandURL2")
}
return resultURL, nil
}
// GetPostURL GetPostURL
func GetPostURL(htmlNode gokogirixml.Node) (string, error) {
//s := QuerySelector(doc, "a._5pcq")
//
//url, ok := s.Attr("href")
//if !ok {
// //TODO: Fix and check all URLs
// rxRelaxed := xurls.Relaxed()
// url = rxRelaxed.FindString(doc.Text())
//
// if len(url) == 0 {
// return "", errors.New("cannot find post url")
// }
//} else {
// url = "https://www.facebook.com" + url
//}
results, err := htmlNode.Search(xPathPostingURL)
if err != nil {
return "", err
}
var genURL string
var attrib *gokogirixml.AttributeNode
if len(results) > 0 {
attrib = results[0].Attribute("href")
//fmt.Printf("%#v\n\n", attrib)
if attrib != nil {
tmpPostURL := attrib.Value()
//fmt.Printf("%#v\n", tmpPostURL)
tmpPostURL = html.UnescapeString(tmpPostURL)
//fmt.Printf("%#v\n", tmpPostURL)
tmpPostURL2, err := url.Parse(tmpPostURL)
//fmt.Printf("%#v -- %#v\n", tmpPostURL2, err)
if err != nil {
attrib = nil
goto nextPostLinkChecker
}
//fmt.Printf("%#v\n", tmpPostURL2)
newQuery := url.Values{}
oldQuery := tmpPostURL2.Query()
newQuery.Set("nx_source", oldQuery.Get("nx_source"))
newQuery.Set("gift_id", oldQuery.Get("gift_id"))
tmpURLGen := url.URL{
Scheme: tmpPostURL2.Scheme,
Host: tmpPostURL2.Host,
Path: `herowarsgame/posts/` + dataFT.TopLevelPostID,
}
if tmpURLGen.Scheme != "https" {
tmpURLGen.Scheme = "https"
}
if tmpURLGen.Host != "www.facebook.com" {
tmpURLGen.Host = "www.facebook.com"
}
// dataFT.
genURL = tmpURLGen.String()
}
//fmt.Printf("GENURL: %#v\n", genURL)
}
nextPostLinkChecker:
resultURL, err := ExpandURL2(genURL, fbPageURL)
if err != nil {
return "", errors.New("cannot find post url - ExpandURL2")
}
return resultURL, nil
}
type object interface {
Find(string) *goquery.Selection
}
// QuerySelector QuerySelector
func QuerySelector(s object, selector string) *goquery.Selection {
return s.Find(selector).First()
}
// QuerySelectorEach QuerySelectorEach
func QuerySelectorEach(s object, selector string, mf func(i int, selection *goquery.Selection)) *goquery.Selection {
return s.Find(selector).Each(mf)
}
// ParseTimeStamp ParseTimeStamp
func ParseTimeStamp(utime int64) (string, error) {
t := time.Unix(utime, 0)
return t.Format(time.RFC3339), nil
}
// GetTimeStamp GetTimeStamp
func GetTimeStamp(htmlNode gokogirixml.Node) (string, error) {
//s := QuerySelector(doc, "._5ptz.timestamp.livetimestamp")
//s := QuerySelector(doc, "abbr._5ptz")
results, err := htmlNode.Search(xPathArticle)
if err != nil {
return "", err
}
if attrib := results[0].Attribute("data-ft"); attrib != nil {
json := attrib.Value()
normalJSON := html.UnescapeString(json)
dataFTinternal, err := UnmarshalDataFT([]byte(normalJSON))
dataFT = &dataFTinternal
if err != nil {
return "", err
}
pid := dataFTinternal.PageID
return ParseTimeStamp(dataFTinternal.PageInsights[pid].PostContext.PublishTime)
}
return "", errors.New("cannot find timestamp")
}
// ProfileLink ProfileLink
type ProfileLink struct {
Name string
URL string
}
// GetProfileLink GetProfileLink
func GetProfileLink(doc *goquery.Document) (*ProfileLink, error) {
s := QuerySelector(doc, "a.profileLink")
if s.Length() == 0 {
s = QuerySelector(doc, "span.fwb.fcg > a")
}
pl := ProfileLink{}
pl.Name = s.Text()
if pl.Name == "" {
return nil, errors.New("cannot find name of profile link")
}
url, ok := s.Attr("href")
if !ok {
return nil, errors.New("cannot find url of profile link")
}
pl.URL = url
return &pl, nil
}
// GetBlogspotTimeStamp GetBlogspotTimeStamp
func GetBlogspotTimeStamp(doc *goquery.Document) (string, error) {
abbr := QuerySelector(doc, "a.timestamp-link > abbr")
t, ok := abbr.Attr("title")
if ok {
return t, nil
}
return "", errors.New("cannot find timestamp")
}
// GetBlogspotTitle GetBlogspotTitle
func GetBlogspotTitle(doc *goquery.Document) (string, error) {
t := QuerySelector(doc, "h3.post-title")
return strings.TrimSpace(t.Text()), nil
}
//GetBlogspotContent GetBlogspotContent
func GetBlogspotContent(doc *goquery.Document) (string, error) {
c := QuerySelector(doc, "div.post-body")
s, err := c.Html()
if err != nil {
return "", err
}
var lines []string
scanner := bufio.NewScanner(strings.NewReader(s))
for scanner.Scan() {
lines = append(lines, " "+scanner.Text())
}
if err := scanner.Err(); err != nil {
return "", err
}
return strings.Join(lines, "\n"), nil
}
//GetBlogspotURL GetBlogspotURL
func GetBlogspotURL(doc *goquery.Document) (string, error) {
meta := QuerySelector(doc, "meta[property='og:url']")
u, ok := meta.Attr("content")
if ok {
return u, nil
}
return "", errors.New("cannot find url")
}
// GetBlogspotSummary GetBlogspotSummary
func GetBlogspotSummary(doc *goquery.Document) (string, error) {
meta := QuerySelector(doc, "meta[property='og:description']")
d, ok := meta.Attr("content")
if ok {
return strings.TrimSpace(d), nil
}
return "", errors.New("cannot find summary")
}
// GetBlogspotAuthor GetBlogspotAuthor
func GetBlogspotAuthor(doc *goquery.Document) (string, error) {
a := QuerySelector(doc, "span.post-author > span.fn")
return a.Text(), nil
}
//GetBlogspotTags GetBlogspotTags
func GetBlogspotTags(doc *goquery.Document) (string, error) {
s := doc.Find("span.post-labels > a")
labels := ""
s.Each(func(_ int, l *goquery.Selection) {
if labels != "" {
labels += ", "
}
labels += l.Text()
})
return labels, nil
}
// ParseBlogspotPost ParseBlogspotPost
func ParseBlogspotPost(doc *goquery.Document) (*FBPostData, error) {
bs := FBPostData{}
var err error
bs.TimeStamp, err = GetBlogspotTimeStamp(doc)
if err != nil {
return &bs, err
}
bs.Title, err = GetBlogspotTitle(doc)
if err != nil {
return &bs, err
}
bs.Content, err = GetBlogspotContent(doc)
if err != nil {
return &bs, err
}
bs.PostURL, err = GetBlogspotURL(doc)
if err != nil {
return &bs, err
}
bs.Summary, err = GetBlogspotSummary(doc)
if err != nil {
return &bs, err
}
bs.Author, err = GetBlogspotAuthor(doc)
if err != nil {
return &bs, err
}
bs.Tags, err = GetBlogspotTags(doc)
if err != nil {
return &bs, err
}
return &bs, nil
}

64
fb_dataft.go Normal file
View File

@@ -0,0 +1,64 @@
// This file was generated from JSON Schema using quicktype, do not modify it directly.
// To parse and unparse this JSON data, add this code to your project and do:
//
// dataFT, err := UnmarshalDataFT(bytes)
// bytes, err = dataFT.Marshal()
package main
import "encoding/json"
func UnmarshalDataFT(data []byte) (DataFT, error) {
var r DataFT
err := json.Unmarshal(data, &r)
return r, err
}
func (r *DataFT) Marshal() ([]byte, error) {
return json.Marshal(r)
}
type DataFT struct {
MFStoryKey string `json:"mf_story_key"`
TopLevelPostID string `json:"top_level_post_id"`
TlObjid string `json:"tl_objid"`
ContentOwnerIDNew string `json:"content_owner_id_new"`
ThrowbackStoryFbid string `json:"throwback_story_fbid"`
PageID string `json:"page_id"`
PhotoID string `json:"photo_id"`
StoryLocation int64 `json:"story_location"`
StoryAttachmentStyle string `json:"story_attachment_style"`
PageInsights map[string]PageInsight `json:"page_insights"`
Tn string `json:"tn"`
}
type PageInsight struct {
PageID string `json:"page_id"`
ActorID string `json:"actor_id"`
Dm Dm `json:"dm"`
Psn string `json:"psn"`
PostContext PostContext `json:"post_context"`
Role int64 `json:"role"`
Sl int64 `json:"sl"`
Targets []Target `json:"targets"`
}
type Dm struct {
IsShare int64 `json:"isShare"`
OriginalPostOwnerID int64 `json:"originalPostOwnerID"`
}
type PostContext struct {
ObjectFbtype int64 `json:"object_fbtype"`
PublishTime int64 `json:"publish_time"`
StoryName string `json:"story_name"`
StoryFbid []string `json:"story_fbid"`
}
type Target struct {
ActorID string `json:"actor_id"`
PageID string `json:"page_id"`
PostID string `json:"post_id"`
Role int64 `json:"role"`
ShareID int64 `json:"share_id"`
}

20
go.mod Normal file
View File

@@ -0,0 +1,20 @@
module git.deineagentur.com/Jomaar/fbBot
go 1.13
require (
github.com/PuerkitoBio/goquery v1.5.0
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4
github.com/chromedp/chromedp v0.5.2
github.com/fsnotify/fsnotify v1.4.7
github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f
github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546
github.com/mailru/easyjson v0.7.0
github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756
github.com/microcosm-cc/bluemonday v1.0.2
github.com/robfig/cron v1.2.0
github.com/robfig/cron/v3 v3.0.1
github.com/spf13/viper v1.6.1
gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc
mvdan.cc/xurls/v2 v2.1.0
)

179
go.sum Normal file
View File

@@ -0,0 +1,179 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 h1:QD3KxSJ59L2lxG6MXBjNHxiQO2RmxTQ3XcK+wO44WOg=
github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
github.com/chromedp/chromedp v0.5.2 h1:W8xBXQuUnd2dZK0SN/lyVwsQM7KgW+kY5HGnntms194=
github.com/chromedp/chromedp v0.5.2/go.mod h1:rsTo/xRo23KZZwFmWk2Ui79rBaVRRATCjLzNQlOFSiA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f h1:6UIvzqlGM38lOpKP380Wbl0kUyyjutcc7KJUaDM/U4o=
github.com/jbowtie/gokogiri v0.0.0-20190301021639-37f655d3078f/go.mod h1:C3R3VzPq+DAwilxue7DiV6F2QL1rrQX0L56GyI+sBxM=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546 h1:hqxaQP14eTbeZGHZhsDInzj9sJAnEufjVQL4bEA/p+8=
github.com/lunny/html2md v0.0.0-20181018071239-7d234de44546/go.mod h1:lUUaVYlpAQ1Oo6vIZfec6CXQZjOvFZLyqaR8Dl7m+hk=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756 h1:m3Cy2iq0HKujZ4uN59r3XrEZ0ijwcvCOXxgT2pXJEGU=
github.com/maki5/gokogiri v0.0.0-20191003120346-46b5f56d0756/go.mod h1:e8zJNnMchIy3s0Lp9XanSSHmYnGWlbdmBUG81EhrNRU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk=
github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk=
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc h1:LMEBgNcZUqXaP7evD1PZcL6EcDVa2QOFuI+cqM3+AJM=
gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc/go.mod h1:N8UOSI6/c2yOpa/XDz3KVUiegocTziPiqNkeNTMiG1k=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
mvdan.cc/xurls v1.1.0 h1:kj0j2lonKseISJCiq1Tfk+iTv65dDGCl0rTbanXJGGc=
mvdan.cc/xurls/v2 v2.1.0 h1:KaMb5GLhlcSX+e+qhbRJODnUUBvlw01jt4yrjFIHAuA=
mvdan.cc/xurls/v2 v2.1.0/go.mod h1:5GrSd9rOnKOpZaji1OZLYL/yeAAtGDlo/cFe+8K5n8E=

203
main.go Normal file
View File

@@ -0,0 +1,203 @@
package main
import (
"bytes"
"context"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"time"
"github.com/chromedp/chromedp"
"github.com/fsnotify/fsnotify"
"github.com/lunny/html2md"
"github.com/robfig/cron/v3"
"github.com/spf13/viper"
)
var (
// GlobalApp holds a pointer for the App
cronJob *cron.Cron
)
func init() {
viper.SetDefault("development_mode", true)
viper.SetDefault("cronjob_interval", "@every 5m")
viper.SetConfigType("yaml")
viper.SetConfigName("config") // name of config file (without extension)
viper.AddConfigPath("/etc/fbBot/") // path to look for the config file in
viper.AddConfigPath("$HOME/.fbBot") // call multiple times to add many search paths
viper.AddConfigPath(".") // optionally look for config in the working directory
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
// any approach to require this configuration into your program.
var yamlExample = []byte(`
development_mode: true
cronjob_interval: "@every 5m"
no_run_on_start: false
http_useragent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19
(KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19
webhooks:
live:
- https://discordapp.com/api/webhooks/...
- https://discordapp.com/api/webhooks/...
devel:
- https://discordapp.com/api/webhooks/...
comment_on_posts:
- TY
- ty!
- danke!
- I could use it!
- thx!
- Thank you!
- Merci
- Yeah!
- Vielen Dank!
login:
email: example@example.com
passwd: thefbpassword
`)
viper.ReadConfig(bytes.NewBuffer(yamlExample))
viper.WriteConfig()
} else {
// Config file was found but another error was produced
}
}
viper.WatchConfig()
viper.OnConfigChange(func(e fsnotify.Event) {
fmt.Println("Config file changed:", e.Name)
})
html2md.AddRule("span", &html2md.Rule{
Patterns: []string{"span"},
Tp: html2md.Void,
Replacement: func(innerHTML string, attrs []string) string {
return innerHTML
},
})
}
func main() {
SetupCloseHandler()
if viper.GetBool("no_run_on_start") {
go cronTask()
}
cronJob = cron.New()
cronJob.AddFunc(viper.GetString("cronjob_interval"), func() {
cronTask()
})
cronJob.Start()
for {
time.Sleep(30 * time.Second)
}
}
func postRequest(content *Webhook) {
webhooks := viper.GetStringSlice("webhooks.live")
if viper.GetBool("development_mode") {
webhooks = viper.GetStringSlice("webhooks.devel")
}
jsonStrBytes, err := content.Marshal()
if err != nil {
log.Println("postRequest error:", err.Error())
return
}
for _, webhook := range webhooks {
//fmt.Println("URL:>", webhook)
req, err := http.NewRequest("POST", webhook, bytes.NewBuffer(jsonStrBytes))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Println("ERROR: " + err.Error())
}
defer resp.Body.Close()
log.Println("response Status:", resp.Status)
//fmt.Println("response Headers:", resp.Header)
}
}
func cronTask() error {
dir := filepath.Join(".", "content")
os.MkdirAll(dir, os.ModePerm)
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.DisableGPU,
chromedp.UserAgent(viper.GetString("http_useragent")),
chromedp.UserDataDir(dir),
)
allocCtx, cancelAll := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancelAll()
// also set up a custom logger
taskCtx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf), chromedp.WithErrorf(log.Printf))
defer cancel()
chromedp.Run(taskCtx)
page1(taskCtx)
page2(taskCtx)
page3(taskCtx)
page4(taskCtx)
page5(taskCtx)
page6(taskCtx)
page7(taskCtx)
postID := ""
var fbPost *FBPostData
var fbPostErr error
postID, fbPost, fbPostErr = page8(taskCtx)
fmt.Printf("Error: %#v\n", fbPostErr)
if fbPost.TimeStamp != lastUpdatePosted {
page9(taskCtx, postID)
}
time.Sleep(2 * time.Second)
if fbPost.TimeStamp != lastUpdatePosted {
timeStamp := fbPost.TimeStamp
pTime, err := time.Parse(time.RFC3339, fbPost.TimeStamp)
if err == nil {
timeStamp = pTime.UTC().Format("2006-01-02 15:04:05 UTC")
}
embeded := []*Embed{}
embeded = append(embeded, &Embed{
Title: fbPost.Title,
Description: fbPost.Content + `
Date of Post: ` + timeStamp + `
Post URL: ` + fbPost.PostURL,
Image: Image{
URL: fbPost.ImageURL,
},
URL: fbPost.GiftURL,
})
dcontent := &Webhook{
Content: fbPost.Title,
Embeds: embeded,
}
if fbPostErr == nil {
postRequest(dcontent)
}
lastUpdatePosted = fbPost.TimeStamp
viper.Set("lastPostTimestamp", fbPost.TimeStamp)
viper.WriteConfig()
}
return nil
}

75
main_test.go Normal file
View File

@@ -0,0 +1,75 @@
package main
import (
"fmt"
"html"
"io/ioutil"
"net/url"
"strings"
"testing"
)
var (
testStyleURL string = `https\3a //scontent.fbre2-1.fna.fbcdn.net/v/t1.0-9/cp0/e15/q65/s720x720/82084740_2291078311193333_4974989532400189440_o.jpg?_nc_cat\3d 1\26 efg\3d eyJpIjoidCJ9\26 _nc_ohc\3d YFSkDqncwkYAX9wymCm\26 _nc_ht\3d scontent.fbre2-1.fna\26 oh\3d a31ebd3e26518967fa5222b85098d300\26 oe\3d 5EAF6A0D`
testMoopURL string = `/story.php?story_fbid=2291080917859739&amp;id=1574763202824851&amp;_ft_=mf_story_key.2291080917859739%3Atop_level_post_id.2291080917859739%3Atl_objid.2291080917859739%3Acontent_owner_id_new.1574763202824851%3Athrowback_story_fbid.2291080917859739%3Apage_id.1574763202824851%3Aphoto_id.2291078307860000%3Astory_location.4%3Astory_attachment_style.photo%3Apage_insights.%7B%221574763202824851%22%3A%7B%22page_id%22%3A1574763202824851%2C%22actor_id%22%3A1574763202824851%2C%22dm%22%3A%7B%22isShare%22%3A0%2C%22originalPostOwnerID%22%3A0%7D%2C%22psn%22%3A%22EntStatusCreationStory%22%2C%22post_context%22%3A%7B%22object_fbtype%22%3A266%2C%22publish_time%22%3A1578841213%2C%22story_name%22%3A%22EntStatusCreationStory%22%2C%22story_fbid%22%3A%5B2291080917859739%5D%7D%2C%22role%22%3A1%2C%22sl%22%3A4%2C%22targets%22%3A%5B%7B%22actor_id%22%3A1574763202824851%2C%22page_id%22%3A1574763202824851%2C%22post_id%22%3A2291080917859739%2C%22role%22%3A1%2C%22share_id%22%3A0%7D%5D%7D%7D&amp;__tn__=-R`
)
func Test_main(t *testing.T) {
genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`)
genURL = strings.ReplaceAll(genURL, `\3d `, `=`)
genURL = strings.ReplaceAll(genURL, `\26 `, `&`)
fmt.Printf("%#v\n", genURL)
}
func TestCronTask(t *testing.T) {
fmt.Printf("%#v", cronTask())
}
func TestStartTask(t *testing.T) {
fmt.Printf("%#v", startTask())
}
func Test_moop(t *testing.T) {
genURL, err := url.Parse(html.UnescapeString(testMoopURL))
if err != nil {
t.Errorf("%#v", err.Error())
}
fmt.Printf("%#v\n", genURL)
for i, query := range genURL.Query() {
fmt.Printf("%s : %#v\n", i, query)
if i == "_ft_" {
text := ""
for _, txt := range query {
text = text + txt
}
for in, mtext := range strings.SplitN(text, ":", 10) {
allText := strings.Split(mtext, ".")
fmt.Printf("%d ::: %s : %s\n", in, allText[0], allText[1])
}
}
}
}
func TestParse(t *testing.T) {
url := "https://www.facebook.com/herowarsgame/posts/"
//url := "https://www.facebook.com/jayasaro.panyaprateep.org/posts/1095007907274561:0"
//url := "https://www.facebook.com/jayasaro.panyaprateep.org/photos/a.318290164946343.68815.318196051622421/1119561951485823/?type=3"
//result, err := Parse(url)
dat, err := ioutil.ReadFile("./2020-01-14T17:07:58+01:00-PAGE8-content.html")
if err != nil {
t.Error(err)
return
}
//fmt.Print(string(dat))
post, err := ParsePost(string(dat), url)
if err != nil {
t.Error(err)
return
}
t.Log(post)
fmt.Printf("%#v", post)
}

272
page_call.go Normal file
View File

@@ -0,0 +1,272 @@
package main
import (
"context"
"encoding/json"
"log"
"math/rand"
"time"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/chromedp"
"github.com/spf13/viper"
)
func page1(taskCtx context.Context) {
page := "PAGE1"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
// ensure that the browser process is started
if err := chromedp.Run(c,
network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(headers)),
chromedp.Navigate("https://m.facebook.com/"),
//chromedp.Nodes(`document`, &nodes, chromedp.ByJSPath),
chromedp.WaitVisible(emailField),
chromedp.WaitVisible(passwordField),
chromedp.SendKeys(emailField, viper.GetString("login.email")),
chromedp.SendKeys(passwordField, viper.GetString("login.passwd")),
chromedp.WaitReady(`//*[@id="signup-button"]`),
chromedp.Click(`//*[@name="login"]`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page2(taskCtx context.Context) {
return // currently not need
page := "PAGE2"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
// ensure that the browser process is started
if err := chromedp.Run(c,
network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(headers)),
chromedp.Navigate("https://m.facebook.com/"),
//chromedp.Nodes(`document`, &nodes, chromedp.ByJSPath),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page3(taskCtx context.Context) {
page := "PAGE3"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
if err := chromedp.Run(c,
chromedp.WaitVisible(`div._5xu4 > form`),
chromedp.Submit(`div._5xu4 > form`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page4(taskCtx context.Context) {
page := "PAGE4"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
if err := chromedp.Run(c,
chromedp.WaitVisible(`#u_0_0`),
chromedp.SendKeys(`#u_0_0`, viper.GetString("login.passwd")),
chromedp.WaitVisible(`div._4g34 > form`),
chromedp.Submit(`div._4g34 > form`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page5(taskCtx context.Context) {
page := "PAGE5"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
if err := chromedp.Run(c,
chromedp.WaitVisible(`div._4g34 > div._2pii > div > a`),
chromedp.Click(`div._4g34 > div._2pii > div > a`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShotOnly(taskCtx, page)
}
func page6(taskCtx context.Context) {
page := "PAGE6"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
// ensure that the browser process is started
if err := chromedp.Run(c,
network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(headers)),
chromedp.Navigate("https://m.facebook.com/herowarsgame/posts/"),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page7(taskCtx context.Context) {
page := "PAGE7"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second)
defer cancelCtxWTO()
// ensure that the browser process is started
if err := chromedp.Run(c,
chromedp.WaitNotPresent(`a._15ko._77li.touchable._77la`),
chromedp.WaitVisible(`a._15ko._77li.touchable`),
chromedp.Click(`a._15ko._77li.touchable`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
makeScreenShot(taskCtx, page)
}
func page8(taskCtx context.Context) (postID string, fbPost *FBPostData, err error) {
page := "PAGE8"
fbPost = nil
err = nil
c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second)
defer cancelCtxWTO()
const function1 = `(function(d) {
var b = d.evaluate('//*/div[@id="pages_msite_body_contents"]/div/div[3]/div[1]/div/article', document, null, XPathResult.ANY_TYPE, null );
let bb = b.iterateNext();
let json = JSON.parse(bb.dataset.ft)
console.log(json)
return json.top_level_post_id
})(document);`
// ensure that the browser process is started
if err := chromedp.Run(c,
chromedp.WaitVisible(`a._15ko._77li.touchable`),
chromedp.ActionFunc(func(ctx context.Context) error {
tmpPostID, exp, err := runtime.Evaluate(function1).Do(ctx)
if err != nil {
log.Println(err)
return err
}
if exp != nil {
log.Println(exp)
return exp
}
err = json.Unmarshal(tmpPostID.Value, &postID)
if err != nil {
log.Println(err)
return err
}
return nil
}),
chromedp.Sleep(1*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return "", nil, err
//panic(err)
}
fbPost, err = makeScreenShotAndParsePost(taskCtx, page)
return postID, fbPost, err
}
func page9(taskCtx context.Context, postID string) {
page := "PAGE9"
c, cancelCtxWTO := context.WithTimeout(taskCtx, 15*time.Second)
defer cancelCtxWTO()
clickSel := `//*[@id="composer-` + postID + `"]/form/div[1]/div[3]/button`
//formSel := `//*[@id="composer-` + postID + `"]/form`
if postID != "" {
// ensure that the browser process is started
if err := chromedp.Run(c,
chromedp.WaitVisible(`//*[@id="feedback_inline_`+postID+`"]/div[2]/div[2]/a`),
chromedp.Click(`//*[@id="feedback_inline_`+postID+`"]/div[2]/div[2]/a`),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
return
//panic(err)
}
}
makeScreenShot(taskCtx, page)
page = page + "-1"
// ensure that the browser process is started
if err := chromedp.Run(c,
chromedp.WaitVisible(`//*/textarea[@id="composerInput"]`),
chromedp.SendKeys(`//*/textarea[@id="composerInput"]`, sendRandomText()),
chromedp.Sleep(time.Second),
chromedp.WaitVisible(clickSel),
chromedp.WaitEnabled(clickSel),
chromedp.Click(clickSel),
//chromedp.Sleep(2*time.Second),
chromedp.Sleep(2*time.Second),
); err != nil {
log.Println(page)
log.Println(err)
panic(err)
//return
}
makeScreenShot(taskCtx, page)
}
func sendRandomText() string {
s := rand.NewSource(time.Now().Unix())
r := rand.New(s) // initialize local pseudorandom generator
return randomText[r.Intn(len(randomText))]
}

175
screenshot.go Normal file
View File

@@ -0,0 +1,175 @@
package main
import (
"context"
"io/ioutil"
"math"
"time"
"github.com/spf13/viper"
"github.com/chromedp/cdproto/emulation"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
func makeScreenShot(c context.Context, page string) {
timeNow := time.Now().Format(time.RFC3339)
var buf []byte
var content string
// capture entire browser viewport, returning png with quality=90
if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil {
//log.Fatal(err)
}
if viper.GetBool("development_mode") {
if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
//log.Fatal(err)
}
if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil {
//log.Fatal(err)
}
}
}
func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, error) {
timeNow := time.Now().Format(time.RFC3339)
var buf []byte
var content string
// capture entire browser viewport, returning png with quality=90
if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil {
//log.Fatal(err)
}
if viper.GetBool("development_mode") {
if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
//log.Fatal(err)
}
if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil {
//log.Fatal(err)
}
}
return ParsePost(content, "https://www.facebook.com/pg/herowarsgame/posts/?ref=page_internal")
}
func makeScreenShotOnly(c context.Context, page string) {
timeNow := time.Now().Format(time.RFC3339)
var buf []byte
// capture entire browser viewport, returning png with quality=90
if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil {
//log.Fatal(err)
}
if viper.GetBool("development_mode") {
if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
//log.Fatal(err)
}
}
}
// fullScreenshot takes a screenshot of the entire browser viewport.
//
// Liberally copied from puppeteer's source.
//
// Note: this will override the viewport emulation settings.
func fullScreenshot(quality int64, res *[]byte, content *string) chromedp.Tasks {
return chromedp.Tasks{
chromedp.Sleep(1 * time.Second),
chromedp.ActionFunc(func(ctx context.Context) error {
// get layout metrics
_, _, contentSize, err := page.GetLayoutMetrics().Do(ctx)
if err != nil {
return err
}
width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height))
// force viewport emulation
err = emulation.SetDeviceMetricsOverride(width, height, 1, false).
WithScreenOrientation(&emulation.ScreenOrientation{
Type: emulation.OrientationTypePortraitPrimary,
Angle: 0,
}).
Do(ctx)
if err != nil {
return err
}
// capture screenshot
*res, err = page.CaptureScreenshot().
WithQuality(quality).
WithClip(&page.Viewport{
X: contentSize.X,
Y: contentSize.Y,
Width: contentSize.Width,
Height: contentSize.Height,
Scale: 1,
}).Do(ctx)
if err != nil {
return err
}
return nil
}),
chromedp.OuterHTML("html", content),
}
}
// fullScreenshotOnly takes a screenshot of the entire browser viewport.
//
// Liberally copied from puppeteer's source.
//
// Note: this will override the viewport emulation settings.
func fullScreenshotOnly(quality int64, res *[]byte) chromedp.Tasks {
return chromedp.Tasks{
chromedp.Sleep(2 * time.Second),
chromedp.ActionFunc(func(ctx context.Context) error {
// get layout metrics
_, _, contentSize, err := page.GetLayoutMetrics().Do(ctx)
if err != nil {
return err
}
width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height))
// force viewport emulation
err = emulation.SetDeviceMetricsOverride(width, height, 1, false).
WithScreenOrientation(&emulation.ScreenOrientation{
Type: emulation.OrientationTypePortraitPrimary,
Angle: 0,
}).
Do(ctx)
if err != nil {
return err
}
// capture screenshot
*res, err = page.CaptureScreenshot().
WithQuality(quality).
WithClip(&page.Viewport{
X: contentSize.X,
Y: contentSize.Y,
Width: contentSize.Width,
Height: contentSize.Height,
Scale: 1,
}).Do(ctx)
if err != nil {
return err
}
return nil
}),
}
}

66
urlexpander.go Normal file
View File

@@ -0,0 +1,66 @@
package main
import (
"net/http"
"strings"
)
// ExpandURL ExpandURL
func ExpandURL(url string) (string, error) {
expandedURL := url
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
expandedURL = req.URL.String()
return nil
},
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
return expandedURL, nil
}
func ExpandURL2(myURL, searchURL string) (string, error) {
nextURL := myURL
var i int
for i < 100 {
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}}
resp, err := client.Get(nextURL)
if err != nil {
return "", err
}
//fmt.Println("StatusCode:", resp.StatusCode)
//fmt.Println(resp.Request.URL)
if resp.StatusCode == 200 {
//fmt.Println("Done!")
break
} else {
nextURLTest := resp.Header.Get("Location")
if strings.Contains(nextURL, searchURL) {
break
}
nextURL = nextURLTest
i++
}
}
return nextURL, nil
}

54
webhook.go Normal file
View File

@@ -0,0 +1,54 @@
package main
import "encoding/json"
func UnmarshalWebhook(data []byte) (Webhook, error) {
var r Webhook
err := json.Unmarshal(data, &r)
return r, err
}
func (r *Webhook) Marshal() ([]byte, error) {
return json.Marshal(r)
}
type Webhook struct {
Username string `json:"username,omitempty"`
AvatarURL string `json:"avatar_url,omitempty"`
TTS string `json:"tts,omitempty"`
Content string `json:"content,omitempty"`
Embeds []*Embed `json:"embeds,omitempty"`
}
type Embed struct {
Author Author `json:"author,omitempty"`
Title string `json:"title,omitempty"`
URL string `json:"url,omitempty"`
Description string `json:"description,omitempty"`
Color int64 `json:"color,omitempty"`
Fields []Field `json:"fields,omitempty"`
Thumbnail Image `json:"thumbnail,omitempty"`
Image Image `json:"image,omitempty"`
Footer Footer `json:"footer,omitempty"`
}
type Author struct {
Name string `json:"name"`
URL string `json:"url"`
IconURL string `json:"icon_url"`
}
type Field struct {
Name string `json:"name"`
Value string `json:"value"`
Inline *bool `json:"inline,omitempty"`
}
type Footer struct {
Text string `json:"text"`
IconURL string `json:"icon_url"`
}
type Image struct {
URL string `json:"url"`
}