hw-fbbot/content_functions.go

package main

import (
	"context"
	"io/ioutil"
	"math"
	"net/http"
	"strings"
	"time"

	"github.com/spf13/viper"

	"github.com/chromedp/cdproto/emulation"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/chromedp"
)

func makeScreenShot(c context.Context, page string) {

	timeNow := time.Now().Format(time.RFC3339)

	var buf []byte
	var content string
	if viper.GetBool("development_mode") {
		// capture entire browser viewport, returning png with quality=90
		if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil {
			//log.Fatal(err)
		}

		if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
			//log.Fatal(err)
		}

		if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil {
			//log.Fatal(err)
		}
	}

}

func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, error) {

	timeNow := time.Now().Format(time.RFC3339)

	var buf []byte
	var content string

	if viper.GetBool("development_mode") {
		// capture entire browser viewport, returning png with quality=90
		if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil {
			//log.Fatal(err)
		}

		if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
			//log.Fatal(err)
		}

		if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil {
			//log.Fatal(err)
		}
	}
	return ParsePost(content, "https://www.facebook.com/pg/herowarsgame/posts/?ref=page_internal")

}

func makeScreenShotOnly(c context.Context, page string) {

	timeNow := time.Now().Format(time.RFC3339)

	var buf []byte
	if viper.GetBool("development_mode") {
		// capture entire browser viewport, returning png with quality=90
		if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil {
			//log.Fatal(err)
		}

		if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil {
			//log.Fatal(err)
		}
	}

}

// fullScreenshot takes a screenshot of the entire browser viewport.
//
// Liberally copied from puppeteer's source.
//
// Note: this will override the viewport emulation settings.
func fullScreenshot(quality int64, res *[]byte, content *string) chromedp.Tasks {
	return chromedp.Tasks{
		chromedp.Sleep(1 * time.Second),
		chromedp.ActionFunc(func(ctx context.Context) error {
			// get layout metrics
			_, _, contentSize, err := page.GetLayoutMetrics().Do(ctx)
			if err != nil {
				return err
			}

			width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height))

			// force viewport emulation
			err = emulation.SetDeviceMetricsOverride(width, height, 1, false).
				WithScreenOrientation(&emulation.ScreenOrientation{
					Type:  emulation.OrientationTypePortraitPrimary,
					Angle: 0,
				}).
				Do(ctx)
			if err != nil {
				return err
			}

			// capture screenshot
			*res, err = page.CaptureScreenshot().
				WithQuality(quality).
				WithClip(&page.Viewport{
					X:      contentSize.X,
					Y:      contentSize.Y,
					Width:  contentSize.Width,
					Height: contentSize.Height,
					Scale:  1,
				}).Do(ctx)
			if err != nil {
				return err
			}
			return nil
		}),
		chromedp.OuterHTML("html", content),
	}
}

// fullScreenshotOnly takes a screenshot of the entire browser viewport.
//
// Liberally copied from puppeteer's source.
//
// Note: this will override the viewport emulation settings.
func fullScreenshotOnly(quality int64, res *[]byte) chromedp.Tasks {
	return chromedp.Tasks{
		chromedp.Sleep(2 * time.Second),
		chromedp.ActionFunc(func(ctx context.Context) error {
			// get layout metrics
			_, _, contentSize, err := page.GetLayoutMetrics().Do(ctx)
			if err != nil {
				return err
			}

			width, height := int64(math.Ceil(contentSize.Width)), int64(math.Ceil(contentSize.Height))

			// force viewport emulation
			err = emulation.SetDeviceMetricsOverride(width, height, 1, false).
				WithScreenOrientation(&emulation.ScreenOrientation{
					Type:  emulation.OrientationTypePortraitPrimary,
					Angle: 0,
				}).
				Do(ctx)
			if err != nil {
				return err
			}

			// capture screenshot
			*res, err = page.CaptureScreenshot().
				WithQuality(quality).
				WithClip(&page.Viewport{
					X:      contentSize.X,
					Y:      contentSize.Y,
					Width:  contentSize.Width,
					Height: contentSize.Height,
					Scale:  1,
				}).Do(ctx)
			if err != nil {
				return err
			}
			return nil
		}),
	}
}

// ExpandURL ExpandURL
func ExpandURL(url string) (string, error) {
	expandedURL := url

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			expandedURL = req.URL.String()
			return nil
		},
	}

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return "", err
	}

	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	return expandedURL, nil
}

// ExpandURL2 ExpandURL2
func ExpandURL2(myURL, searchURL string) (string, error) {
	nextURL := myURL
	var i int
	for i < 100 {
		client := &http.Client{
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			}}

		resp, err := client.Get(nextURL)

		if err != nil {
			return "", err
		}

		//fmt.Println("StatusCode:", resp.StatusCode)
		//fmt.Println(resp.Request.URL)

		if resp.StatusCode == 200 {
			//fmt.Println("Done!")
			break
		} else {
			nextURLTest := resp.Header.Get("Location")
			if strings.Contains(nextURL, searchURL) {
				break
			}
			nextURL = nextURLTest
			i++
		}
	}

	return nextURL, nil
}

func getImageURL(testStyleURL string) (string, error) {
	genURL := strings.ReplaceAll(testStyleURL, `\3a `, `:`)
	genURL = strings.ReplaceAll(genURL, `\3d `, `=`)
	genURL = strings.ReplaceAll(genURL, `\26 `, `&`)
	return genURL, nil
}