diff --git a/content_functions.go b/content_functions.go index d2a7ea9..bdd0ddb 100644 --- a/content_functions.go +++ b/content_functions.go @@ -1,6 +1,241 @@ package main -import "strings" +import ( + "context" + "io/ioutil" + "math" + "net/http" + "strings" + "time" + + "github.com/spf13/viper" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" +) + +func makeScreenShot(c context.Context, page string) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + var content string + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + //log.Fatal(err) + } + } + +} + +func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, error) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + var content string + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + //log.Fatal(err) + } + } + return ParsePost(content, "https://www.facebook.com/pg/herowarsgame/posts/?ref=page_internal") + +} + +func makeScreenShotOnly(c context.Context, page string) { + + timeNow := time.Now().Format(time.RFC3339) + + var buf []byte + + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil { + //log.Fatal(err) + } + + if viper.GetBool("development_mode") { + if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + } + +} + +// fullScreenshot takes a screenshot of the entire browser viewport. +// +// Liberally copied from puppeteer's source. +// +// Note: this will override the viewport emulation settings. +func fullScreenshot(quality int64, res *[]byte, content *string) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Sleep(1 * time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + // get layout metrics + _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + // force viewport emulation + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + // capture screenshot + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + chromedp.OuterHTML("html", content), + } +} + +// fullScreenshotOnly takes a screenshot of the entire browser viewport. +// +// Liberally copied from puppeteer's source. +// +// Note: this will override the viewport emulation settings. +func fullScreenshotOnly(quality int64, res *[]byte) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Sleep(2 * time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + // get layout metrics + _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) + if err != nil { + return err + } + + width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) + + // force viewport emulation + err = emulation.SetDeviceMetricsOverride(width, height, 1, false). + WithScreenOrientation(&emulation.ScreenOrientation{ + Type: emulation.OrientationTypePortraitPrimary, + Angle: 0, + }). + Do(ctx) + if err != nil { + return err + } + + // capture screenshot + *res, err = page.CaptureScreenshot(). + WithQuality(quality). + WithClip(&page.Viewport{ + X: contentSize.X, + Y: contentSize.Y, + Width: contentSize.Width, + Height: contentSize.Height, + Scale: 1, + }).Do(ctx) + if err != nil { + return err + } + return nil + }), + } +} + +// ExpandURL ExpandURL +func ExpandURL(url string) (string, error) { + expandedURL := url + + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + expandedURL = req.URL.String() + return nil + }, + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + return expandedURL, nil +} + +// ExpandURL2 ExpandURL2 +func ExpandURL2(myURL, searchURL string) (string, error) { + nextURL := myURL + var i int + for i < 100 { + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }} + + resp, err := client.Get(nextURL) + + if err != nil { + return "", err + } + + //fmt.Println("StatusCode:", resp.StatusCode) + //fmt.Println(resp.Request.URL) + + if resp.StatusCode == 200 { + //fmt.Println("Done!") + break + } else { + nextURLTest := resp.Header.Get("Location") + if strings.Contains(nextURL, searchURL) { + break + } + nextURL = nextURLTest + i++ + } + } + + return nextURL, nil +} func getImageURL(testStyleURL string) (string, error) { genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`) diff --git a/main.go b/main.go index 197d83a..d7ce595 100644 --- a/main.go +++ b/main.go @@ -34,19 +34,11 @@ cronjob_interval: "@every 5m" no_run_on_start: false http_useragent: "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19" webhooks: -<<<<<<< HEAD - live: - - https://discordapp.com/api/webhooks/... - - https://discordapp.com/api/webhooks/... - devel: - - https://discordapp.com/api/webhooks/... -======= live: - https://discordapp.com/api/webhooks/... - https://discordapp.com/api/webhooks/... devel: - https://discordapp.com/api/webhooks/... ->>>>>>> 49d60f1... update some typos and missing data comment_on_posts: - TY - ty! diff --git a/screenshot.go b/screenshot.go deleted file mode 100644 index 16c6885..0000000 --- a/screenshot.go +++ /dev/null @@ -1,175 +0,0 @@ -package main - -import ( - "context" - "io/ioutil" - "math" - "time" - - "github.com/spf13/viper" - - "github.com/chromedp/cdproto/emulation" - "github.com/chromedp/cdproto/page" - "github.com/chromedp/chromedp" -) - -func makeScreenShot(c context.Context, page string) { - - timeNow := time.Now().Format(time.RFC3339) - - var buf []byte - var content string - - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { - //log.Fatal(err) - } - - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { - //log.Fatal(err) - } - - if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { - //log.Fatal(err) - } - } - -} - -func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, error) { - - timeNow := time.Now().Format(time.RFC3339) - - var buf []byte - var content string - - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { - //log.Fatal(err) - } - - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { - //log.Fatal(err) - } - - if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { - //log.Fatal(err) - } - } - return ParsePost(content, "https://www.facebook.com/pg/herowarsgame/posts/?ref=page_internal") - -} - -func makeScreenShotOnly(c context.Context, page string) { - - timeNow := time.Now().Format(time.RFC3339) - - var buf []byte - - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil { - //log.Fatal(err) - } - - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { - //log.Fatal(err) - } - } - -} - -// fullScreenshot takes a screenshot of the entire browser viewport. -// -// Liberally copied from puppeteer's source. -// -// Note: this will override the viewport emulation settings. -func fullScreenshot(quality int64, res *[]byte, content *string) chromedp.Tasks { - return chromedp.Tasks{ - chromedp.Sleep(1 * time.Second), - chromedp.ActionFunc(func(ctx context.Context) error { - // get layout metrics - _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) - if err != nil { - return err - } - - width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) - - // force viewport emulation - err = emulation.SetDeviceMetricsOverride(width, height, 1, false). - WithScreenOrientation(&emulation.ScreenOrientation{ - Type: emulation.OrientationTypePortraitPrimary, - Angle: 0, - }). - Do(ctx) - if err != nil { - return err - } - - // capture screenshot - *res, err = page.CaptureScreenshot(). - WithQuality(quality). - WithClip(&page.Viewport{ - X: contentSize.X, - Y: contentSize.Y, - Width: contentSize.Width, - Height: contentSize.Height, - Scale: 1, - }).Do(ctx) - if err != nil { - return err - } - return nil - }), - chromedp.OuterHTML("html", content), - } -} - -// fullScreenshotOnly takes a screenshot of the entire browser viewport. -// -// Liberally copied from puppeteer's source. -// -// Note: this will override the viewport emulation settings. -func fullScreenshotOnly(quality int64, res *[]byte) chromedp.Tasks { - return chromedp.Tasks{ - chromedp.Sleep(2 * time.Second), - chromedp.ActionFunc(func(ctx context.Context) error { - // get layout metrics - _, _, contentSize, err := page.GetLayoutMetrics().Do(ctx) - if err != nil { - return err - } - - width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height)) - - // force viewport emulation - err = emulation.SetDeviceMetricsOverride(width, height, 1, false). - WithScreenOrientation(&emulation.ScreenOrientation{ - Type: emulation.OrientationTypePortraitPrimary, - Angle: 0, - }). - Do(ctx) - if err != nil { - return err - } - - // capture screenshot - *res, err = page.CaptureScreenshot(). - WithQuality(quality). - WithClip(&page.Viewport{ - X: contentSize.X, - Y: contentSize.Y, - Width: contentSize.Width, - Height: contentSize.Height, - Scale: 1, - }).Do(ctx) - if err != nil { - return err - } - return nil - }), - } -} diff --git a/urlexpander.go b/urlexpander.go deleted file mode 100644 index 0d8ccf3..0000000 --- a/urlexpander.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "net/http" - "strings" -) - -// ExpandURL ExpandURL -func ExpandURL(url string) (string, error) { - expandedURL := url - - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - expandedURL = req.URL.String() - return nil - }, - } - - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return "", err - } - - resp, err := client.Do(req) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - return expandedURL, nil -} - -// ExpandURL2 ExpandURL2 -func ExpandURL2(myURL, searchURL string) (string, error) { - nextURL := myURL - var i int - for i < 100 { - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }} - - resp, err := client.Get(nextURL) - - if err != nil { - return "", err - } - - //fmt.Println("StatusCode:", resp.StatusCode) - //fmt.Println(resp.Request.URL) - - if resp.StatusCode == 200 { - //fmt.Println("Done!") - break - } else { - nextURLTest := resp.Header.Get("Location") - if strings.Contains(nextURL, searchURL) { - break - } - nextURL = nextURLTest - i++ - } - } - - return nextURL, nil -}