From 35de12b3e96839151f7beedfa7e83a0e2d249e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Fr=C3=B6hle?= Date: Wed, 15 Jan 2020 18:00:58 +0100 Subject: [PATCH] some more tweaks and fixes --- .gitignore | 3 +++ Dockerfile | 31 +++++++++++++++++++++++++++++++ configuration.go | 3 --- content_functions.go | 42 ++++++++++++++++++++---------------------- main.go | 7 ++++--- page_call.go | 19 +++++++++++-------- 6 files changed, 69 insertions(+), 36 deletions(-) create mode 100644 Dockerfile diff --git a/.gitignore b/.gitignore index 70ae127..2a45d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ content/ *.html *.png +custom-data/ +config.yaml +tmp/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e716331 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +############################ +# STEP 1 build executable binary +############################ +FROM golang:alpine AS builder +# Install git. +# Git is required for fetching the dependencies. +RUN apk update && \ + apk add --no-cache ca-certificates pkgconf git libxml2-dev oniguruma-dev && \ + apk add --virtual build-dependencies build-base gcc wget git && \ + mkdir -p /opt/build +WORKDIR /opt/build +COPY . . +# Fetch dependencies. +# Using go get. +RUN go get -d -v +# Build the binary. +RUN go build -o /go/bin/fbBot . +############################ +# STEP 2 build a small image +############################ +FROM alpine +VOLUME /opt/tmp +RUN apk update && \ + apk add --no-cache ca-certificates chromium chromium-chromedriver +# Copy our static executable. +COPY --from=builder /go/bin/fbBot /opt/fbBot +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY contrib/ / +COPY config.yaml /etc/fbBot/config.yaml +# Run the hello binary. +ENTRYPOINT ["/opt/fbBot"] \ No newline at end of file diff --git a/configuration.go b/configuration.go index 974346d..c1daf13 100644 --- a/configuration.go +++ b/configuration.go @@ -5,7 +5,6 @@ import ( "github.com/chromedp/cdproto/cdp" "github.com/robfig/cron/v3" - "github.com/spf13/viper" ) const ( @@ -63,8 +62,6 @@ var ( } nodes []*cdp.Node - randomText = viper.GetStringSlice("comment_on_posts") - dataFT *DataFT lastUpdatePosted string diff --git a/content_functions.go b/content_functions.go index bdd0ddb..6f8e6b3 100644 --- a/content_functions.go +++ b/content_functions.go @@ -21,18 +21,17 @@ func makeScreenShot(c context.Context, page string) { var buf []byte var content string - - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { - //log.Fatal(err) - } - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { //log.Fatal(err) } - if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { //log.Fatal(err) } } @@ -46,17 +45,17 @@ func makeScreenShotAndParsePost(c context.Context, page string) (*FBPostData, er var buf []byte var content string - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { - //log.Fatal(err) - } - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshot(90, &buf, &content)); err != nil { //log.Fatal(err) } - if err := ioutil.WriteFile(timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { + if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-content.html", []byte(content), 0644); err != nil { //log.Fatal(err) } } @@ -69,14 +68,13 @@ func makeScreenShotOnly(c context.Context, page string) { timeNow := time.Now().Format(time.RFC3339) var buf []byte - - // capture entire browser viewport, returning png with quality=90 - if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil { - //log.Fatal(err) - } - if viper.GetBool("development_mode") { - if err := ioutil.WriteFile(timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { + // capture entire browser viewport, returning png with quality=90 + if err := chromedp.Run(c, fullScreenshotOnly(90, &buf)); err != nil { + //log.Fatal(err) + } + + if err := ioutil.WriteFile("/opt/tmp/"+timeNow+"-"+page+"-fullScreenshot.png", buf, 0644); err != nil { //log.Fatal(err) } } diff --git a/main.go b/main.go index 8010d73..16f4534 100644 --- a/main.go +++ b/main.go @@ -76,8 +76,9 @@ login: } func main() { + os.MkdirAll("/opt/tmp", os.ModeDir) SetupCloseHandler() - if viper.GetBool("no_run_on_start") { + if !viper.GetBool("no_run_on_start") { go cronTask() } @@ -155,13 +156,13 @@ func cronTask() error { var fbPostErr error postID, fbPost, fbPostErr = page8(taskCtx) fmt.Printf("Error: %#v\n", fbPostErr) - if fbPost.TimeStamp != lastUpdatePosted { + if fbPost != nil && fbPost.TimeStamp != lastUpdatePosted { page9(taskCtx, postID) } time.Sleep(2 * time.Second) - if fbPost.TimeStamp != lastUpdatePosted { + if fbPost != nil && fbPost.TimeStamp != lastUpdatePosted { timeStamp := fbPost.TimeStamp pTime, err := time.Parse(time.RFC3339, fbPost.TimeStamp) if err == nil { diff --git a/page_call.go b/page_call.go index cf492f5..2ea3bca 100644 --- a/page_call.go +++ b/page_call.go @@ -15,7 +15,7 @@ import ( func page1(taskCtx context.Context) { page := "PAGE1" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() // ensure that the browser process is started @@ -45,7 +45,7 @@ func page2(taskCtx context.Context) { return // currently not need page := "PAGE2" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() // ensure that the browser process is started @@ -67,7 +67,7 @@ func page2(taskCtx context.Context) { func page3(taskCtx context.Context) { page := "PAGE3" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() if err := chromedp.Run(c, @@ -86,9 +86,11 @@ func page3(taskCtx context.Context) { func page4(taskCtx context.Context) { page := "PAGE4" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() + makeScreenShot(taskCtx, page+"pre") + if err := chromedp.Run(c, chromedp.WaitVisible(`#u_0_0`), chromedp.SendKeys(`#u_0_0`, viper.GetString("login.passwd")), @@ -108,7 +110,7 @@ func page4(taskCtx context.Context) { func page5(taskCtx context.Context) { page := "PAGE5" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() if err := chromedp.Run(c, @@ -127,7 +129,7 @@ func page5(taskCtx context.Context) { func page6(taskCtx context.Context) { page := "PAGE6" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() // ensure that the browser process is started @@ -148,7 +150,7 @@ func page6(taskCtx context.Context) { func page7(taskCtx context.Context) { page := "PAGE7" - c, cancelCtxWTO := context.WithTimeout(taskCtx, 5*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) defer cancelCtxWTO() // ensure that the browser process is started @@ -172,7 +174,7 @@ func page8(taskCtx context.Context) (postID string, fbPost *FBPostData, err erro fbPost = nil err = nil - c, cancelCtxWTO := context.WithTimeout(taskCtx, 10*time.Second) + c, cancelCtxWTO := context.WithTimeout(taskCtx, 15*time.Second) defer cancelCtxWTO() const function1 = `(function(d) { @@ -265,6 +267,7 @@ func page9(taskCtx context.Context, postID string) { } func sendRandomText() string { + var randomText []string = viper.GetStringSlice("comment_on_posts") s := rand.NewSource(time.Now().Unix()) r := rand.New(s) // initialize local pseudorandom generator return randomText[r.Intn(len(randomText))]