release: v1.7.0

Closes https://gitlab.com/commento/commento/issues/88

.gitlab-ci.yml

@@ -3,14 +3,17 @@ stages:
   - go-fmt
   - go-test
   - build-src
+  - aws-upload-tags
   - build-docker
-  - docker-registry
+  - docker-registry-master
+  - docker-registry-tags
 
 check-dco:
   stage: check-dco
   image: debian:buster
   except:
     - master
+    - tags
   script:
     - apt update
     - apt install -y curl git jq
@@ -19,21 +22,37 @@ check-dco:
 build-src:
   stage: build-src
   image: debian:buster
-  variables:
-    GOPATH: $CI_PROJECT_DIR
   except:
     - master
+    - tags
+  before_script:
+    - bash $CI_PROJECT_DIR/scripts/gitlab-ci-build-prescript
   script:
-    - apt update
-    - apt install -y curl gnupg git make golang
-    - curl -sL https://deb.nodesource.com/setup_10.x | bash -
-    - apt update
-    - apt install -y nodejs
-    - npm install -g html-minifier@3.5.7 uglify-js@3.4.1 sass@1.5.1
-    - mkdir -p src/gitlab.com/commento && cd src/gitlab.com/commento && ln -s $CI_PROJECT_DIR && cd $CI_PROJECT_NAME
+    - export GOPATH=/go
+    - export PATH=$PATH:/go/bin
+    - cd /go/src/$CI_PROJECT_NAME
     - make devel
     - make prod
 
+aws-upload-tags:
+  stage: aws-upload-tags
+  image: debian:buster
+  environment: aws-upload-tags
+  variables:
+    AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+  only:
+    - tags
+  before_script:
+    - bash $CI_PROJECT_DIR/scripts/gitlab-ci-build-prescript
+  script:
+    - export GOPATH=/go
+    - export PATH=$PATH:/go/bin
+    - cd /go/src/$CI_PROJECT_NAME
+    - make prod
+    - cd build/prod && tar -zcvf /commento-linux-amd64-$(git describe --tags).tar.gz .
+    - aws s3 cp /commento-linux-amd64-$(git describe --tags).tar.gz s3://commento-release/
+
 build-docker:
   stage: build-docker
   image: docker:stable
@@ -41,8 +60,9 @@ build-docker:
     - docker:dind
   except:
     - master
+    - tags
   script:
-    - docker build -t commento-ce .
+    - docker build -t commento .
 
 go-test:
   stage: go-test
@@ -54,11 +74,17 @@ go-test:
     POSTGRES_PASSWORD: postgres
     POSTGRES_DB: commento_test
     COMMENTO_POSTGRES: postgres://postgres:postgres@postgres/commento_test?sslmode=disable
-    GOPATH: $CI_PROJECT_DIR
   except:
     - master
+    - tags
+  before_script:
+    - mkdir -p /go/src /go/bin /go/pkg
+    - export GOPATH=/go
+    - export PATH=$PATH:/go/bin
+    - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+    - ln -s $CI_PROJECT_DIR /go/src/$CI_PROJECT_NAME
   script:
-    - mkdir -p src/gitlab.com/commento && cd src/gitlab.com/commento && ln -s $CI_PROJECT_DIR && cd $CI_PROJECT_NAME
+    - cd /go/src/$CI_PROJECT_NAME
     - make test
 
 go-fmt:
@@ -66,20 +92,35 @@ go-fmt:
   image: golang:1.10.2
   except:
     - master
+    - tags
   script:
     - cd api
     - test -z $(go fmt)
 
-docker-registry:
-  stage: docker-registry
+docker-registry-master:
+  stage: docker-registry-master
   image: docker:stable
   services:
     - docker:dind
   only:
-    - master@commento/commento-ce
+    - master@commento/commento
   before_script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
   script:
-    - docker pull registry.gitlab.com/commento/commento-ce:latest || true
-    - docker build --cache-from registry.gitlab.com/commento/commento-ce:latest --tag registry.gitlab.com/commento/commento-ce:latest .
-    - docker push registry.gitlab.com/commento/commento-ce:latest
+    - docker pull registry.gitlab.com/commento/commento:latest || true
+    - docker build --cache-from registry.gitlab.com/commento/commento:latest --tag registry.gitlab.com/commento/commento:latest .
+    - docker push registry.gitlab.com/commento/commento:latest
+
+docker-registry-tags:
+  stage: docker-registry-tags
+  image: docker:stable
+  services:
+    - docker:dind
+  only:
+    - tags
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
+  script:
+    - apk add git
+    - docker build --tag registry.gitlab.com/commento/commento:$(git describe --tags) .
+    - docker push registry.gitlab.com/commento/commento:$(git describe --tags)

Dockerfile

@@ -1,10 +1,11 @@
 # backend build (api server)
 FROM golang:1.10.2-alpine AS api-build
 
-COPY ./api /go/src/commento-ce/api
-WORKDIR /go/src/commento-ce/api
+COPY ./api /go/src/commento/api
+WORKDIR /go/src/commento/api
 
-RUN apk update && apk add bash make git
+RUN apk update && apk add bash make git curl
+RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
 
 RUN make prod -j$(($(nproc) + 1))
 
@@ -12,8 +13,8 @@ RUN make prod -j$(($(nproc) + 1))
 # frontend build (html, js, css, images)
 FROM node:10.3.0-alpine AS frontend-build
 
-COPY ./frontend /commento-ce/frontend/
-WORKDIR /commento-ce/frontend/
+COPY ./frontend /commento/frontend/
+WORKDIR /commento/frontend/
 
 RUN apk update && apk add bash make
 RUN npm install -g html-minifier@3.5.7 uglify-js@3.4.1 sass@1.5.1
@@ -24,8 +25,8 @@ RUN make prod -j$(($(nproc) + 1))
 # templates build
 FROM alpine:3.7 AS templates-build
 
-COPY ./templates /commento-ce/templates
-WORKDIR /commento-ce/templates
+COPY ./templates /commento/templates
+WORKDIR /commento/templates
 
 RUN apk update && apk add bash make
 
@@ -35,8 +36,8 @@ RUN make prod -j$(($(nproc) + 1))
 # db build
 FROM alpine:3.7 AS db-build
 
-COPY ./db /commento-ce/db
-WORKDIR /commento-ce/db
+COPY ./db /commento/db
+WORKDIR /commento/db
 
 RUN apk update && apk add bash make
 
@@ -46,19 +47,20 @@ RUN make prod -j$(($(nproc) + 1))
 # final image
 FROM alpine:3.7
 
-COPY --from=api-build /go/src/commento-ce/api/build/prod/commento-ce /commento-ce/commento-ce
-COPY --from=frontend-build /commento-ce/frontend/build/prod/*.html /commento-ce/
-COPY --from=frontend-build /commento-ce/frontend/build/prod/css/*.css /commento-ce/css/
-COPY --from=frontend-build /commento-ce/frontend/build/prod/js/*.js /commento-ce/js/
-COPY --from=frontend-build /commento-ce/frontend/build/prod/images/* /commento-ce/images/
-COPY --from=templates-build /commento-ce/templates/build/prod/templates/ /commento-ce/templates/
-COPY --from=db-build /commento-ce/db/build/prod/db/ /commento-ce/db/
+COPY --from=api-build /go/src/commento/api/build/prod/commento /commento/commento
+COPY --from=frontend-build /commento/frontend/build/prod/*.html /commento/
+COPY --from=frontend-build /commento/frontend/build/prod/css/*.css /commento/css/
+COPY --from=frontend-build /commento/frontend/build/prod/js/*.js /commento/js/
+COPY --from=frontend-build /commento/frontend/build/prod/images/* /commento/images/
+COPY --from=frontend-build /commento/frontend/build/prod/fonts/* /commento/fonts/
+COPY --from=templates-build /commento/templates/build/prod/templates/ /commento/templates/
+COPY --from=db-build /commento/db/build/prod/db/ /commento/db/
 
 RUN apk update && apk add ca-certificates --no-cache
 
 EXPOSE 8080
 
-WORKDIR /commento-ce/
+WORKDIR /commento/
 
 ENV COMMENTO_BIND_ADDRESS="0.0.0.0"
-ENTRYPOINT ["/commento-ce/commento-ce"]
+ENTRYPOINT ["/commento/commento"]

README.md

@@ -4,41 +4,69 @@
 
 <p align="center"><b>A bloat-free and privacy-focused discussion platform.</b></p>
 
-Commento is a discussion platform that you can embed on your blog, news articles, and any place where you want your readers to add comments. Commento is fast, lightweight, and privacy-focused; we'll never sell your data, show ads, embed third-party tracking scripts, or inject affiliate links.
+### What is Commento?
 
-#### Features
+Commento allows you to foster discussion on your website &ndash; if you have a blog, you can embed Commento if you want your readers to add comments. It's fast and bloat-free, has a modern interface, and is reasonably secure. Unlike most alternatives, Commento is lightweight and privacy-focused; I'll never sell your data, show ads, embed third-party tracking scripts, or inject affiliate links.
 
- - Privacy-focused
- - Super lightweight, allowing for fast pageloads
- - Automatic spam filtering
- - Review and approve or delete comments through the moderation interface
- - Modern interface with a clean design
- - OAuth support (Google login, for example)
- - Custom CSS theming
- - Import from existing services (like Disqus)
- - Completely free and open source (MIT Expat license)
+### Frequently Asked Questions
 
-#### Editions
+**I don't want to install and manage Commento on a server.**  
+You can use [Commento.io](https://commento.io), the cloud version of Commento, where I do the server hosting, updates, and security and performance tuning for you. To make the hosted service self-sustainable, it is not free. You may choose the plan that best matches your financial situation and needs &ndash; all plans have all features.
 
-There are three editions of Commento.
+**What features does Commento have?**  
+Commento comes with a lot of useful features out-of-the-box: rich text support, upvotes and downvotes, automatic spam detection, moderation tools, sticky comments, thread locking, OAuth login, email notifications, and more!
 
- - **Commento Community Edition (CE)** is open source software that's freely available under the MIT license.
- - [**Commento Enterprise Edition (EE)**](https://commento.io/pricing#self-hosted) includes extra features geared towards organizations that want to self-host.
- - [**Commento Hosted**](https://commento.io) is a hosted version of Commento for those who don't want to host and manage servers. This is currently in private beta and you can [add yourself to the waiting list here](https://commento.io).
+**What does Commento look like? Do you have a demo?**  
+Check out [demo.commento.io](https://demo.commento.io) to play around with a live demo of Commento.
 
-#### Installation and Configuration
+**How is Commento different from Disqus, Facebook Comments, and the rest?**  
+Most other products in this space do not respect your privacy; showing adverts is their primary business model and that nearly always comes at the users' cost. There is no free lunch. Commento is also orders of magnitude lighter than alternatives &ndash; while Disqus and Facebook take megabytes of download to load, Commento is just 11 kB.
 
-See our [documentation on how to install Commento](http://docs.commento.io/installation.html) to get started. We offer several ways to install the software, including a Docker image.
+**Is Commento free software?**  
+Yes. Commento is made [freely available](https://gitlab.com/commento/commento) under the [MIT license](https://gitlab.com/commento/commento/blob/master/LICENSE). And it will always stay that way.
 
-Once you've installed the software, you need to configure it with various environment variables before starting the service. To learn more about this, refer to our documentation on [configuring Commento](https://docs.commento.io/configuration.html).
+**Disqus has a free plan. Why is the [cloud version](https://commento.io) not free of cost?**  
+When I say Commento is free, I mean [free as in freedom](https://www.gnu.org/philosophy/free-sw.en.html). The cloud version is not offered free of cost because servers cost money and offering the service for free would not be sustainable. Unlike most alternatives, Commento does not operate on adverts and shady tactics; you're the customer, not the product.
 
-#### Contributing
+**I have nothing to hide. Why should I care about my privacy?**  
+The thing about privacy is that once you give up control over your information, you can't get it back. You may be fine with having your personal information sold to unknown third-parties today, but when your insurance company uses this information against you tomorrow, you'll regret it. And you'll have no recourse to correct this. Read [this Wikipedia article](https://en.wikipedia.org/wiki/Nothing_to_hide_argument) for more information.
 
-Commento is possible only because of its community. If this is your first contribution to Commento, please go through the [development documentation](https://docs.commento.io/contributing.html) before you begin.
+<div><p style="margin: 0px 0px"><b>As a blog owner, why should I worry about my readers' privacy?</b><br>
+Good question. For starters, your readers value their privacy. Not caring about them is disrespectful and you will end up alienating your audience; they won't come back. But even if you ignore this, you have bigger questions to answer:</p>
+<ul>
+ <li><b>Legality</b>: Did you know that Disqus still isn't GDPR-compliant (according to their <a href="https://help.disqus.com/terms-and-policies/privacy-faq" title="At the time of writing (28 December 2018)" rel="nofollow">privacy policy</a>)?</li>
+ <li><b>Security</b>: What happens when a random third-party script is injected into your website?</li>
+ <li><b>Performance</b>: Did you know that half a second increase in page load time results in a 20% decrease in  engagement and site traffic?</li>
+ <li><b>Ownership</b>: Who owns the content when your readers create comments?</li>
+</ul></div>
+
+**Who's behind this? Are you an evil corporation?**  
+My name is <a href="https://adtac.in">Adhityaa</a>, and I created the project. As someone who's still a student, I promise you I'm neither evil nor a corporation. But I'm not the only one &ndash; dozens of people have contributed to the project and Commento would not exist without these wonderful people.
+
+**Okay, how do I get started?**  
+Glad you asked! You have two options &ndash; self-hosting Commento on your own server or using the [cloud  version](https://commento.io). Start [from here](https://docs.commento.io/getting-started/) to decide which option is right for you and proceed from there.
+
+### Installation
+
+See our [documentation on how to install Commento](https://docs.commento.io/installation/) to get started.
+
+### Contributing
+
+Commento is possible only because of its community. If this is your first contribution to Commento, please go through the [documentation](https://docs.commento.io/contributing/) before you begin.
 
 Help will always be given to those who ask for it. We use IRC for chat to collaborate with other developers. You're invited to [hang out with us](https://irc.commento.io) in the `#commento-dev` channel on freenode if you want to contribute to Commento!
 
-#### License
+### Sponsors
+
+Commento development is partially sponsored by [Mozilla](https://mozilla.org) and [DigitalOcean](https://www.digitalocean.com/) independently.
+
+<p align="center">
+<a href="https://www.mozilla.org/en-US/"><img src="https://user-images.githubusercontent.com/7521600/32265838-d05b2d08-bf0a-11e7-92e1-2cb183eae616.png" title="Mozilla" height="40"></a>
+&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
+<a href="https://www.digitalocean.com"><img src="https://user-images.githubusercontent.com/7521600/32265839-d093c7da-bf0a-11e7-8d99-96a940041d06.png" title="DigitalOcean" height="40"></a>
+</p>
+
+### License
 
 ```
 Copyright 2018 Commento, Inc.

api/Gopkg.lock

@@ -9,6 +9,14 @@
   revision = "64a2037ec6be8a4b0c1d1f706ed35b428b989239"
   version = "v0.26.0"
 
+[[projects]]
+  branch = "master"
+  digest = "1:9769b231d8f5ff406a012aa7f293e45ed69d11617832a1c3c7b8c6ce1558a2a1"
+  name = "github.com/adtac/go-akismet"
+  packages = ["akismet"]
+  pruneopts = "UT"
+  revision = "0ca9e1023047c869ecd4bd3c20780511597a4a77"
+
 [[projects]]
   digest = "1:15042ad3498153684d09f393bbaec6b216c8eec6d61f63dff711de7d64ed8861"
   name = "github.com/golang/protobuf"
@@ -17,6 +25,14 @@
   revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
   version = "v1.1.0"
 
+[[projects]]
+  branch = "master"
+  digest = "1:d03d0fae6a7a80e89c540787a69ab6e0d3b773fdb3303c0b3d96a15490c6ef32"
+  name = "github.com/gomodule/oauth1"
+  packages = ["oauth"]
+  pruneopts = "UT"
+  revision = "9a59ed3b0a84f454c260f2f8f82918223fc5630f"
+
 [[projects]]
   digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1"
   name = "github.com/gorilla/context"
@@ -108,10 +124,12 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:bea0314c10bd362ab623af4880d853b5bad3b63d0ab9945c47e461b8d04203ed"
+  digest = "1:341ceeee37101c62dae441691406bf4ecc71bbeb7b424417879fe88d9f88f487"
   name = "golang.org/x/oauth2"
   packages = [
     ".",
+    "github",
+    "gitlab",
     "google",
     "internal",
     "jws",
@@ -143,6 +161,8 @@
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
+    "github.com/adtac/go-akismet/akismet",
+    "github.com/gomodule/oauth1/oauth",
     "github.com/gorilla/handlers",
     "github.com/gorilla/mux",
     "github.com/lib/pq",
@@ -151,7 +171,10 @@
     "github.com/op/go-logging",
     "github.com/russross/blackfriday",
     "golang.org/x/crypto/bcrypt",
+    "golang.org/x/net/html",
     "golang.org/x/oauth2",
+    "golang.org/x/oauth2/github",
+    "golang.org/x/oauth2/gitlab",
     "golang.org/x/oauth2/google",
   ]
   solver-name = "gps-cdcl"

api/Makefile

@@ -7,9 +7,9 @@ PROD_BUILD_DIR         = $(BUILD_DIR)/prod
 GO_SRC_DIR             = .
 GO_SRC_FILES           = $(wildcard $(GO_SRC_DIR)/*.go)
 GO_DEVEL_BUILD_DIR     = $(DEVEL_BUILD_DIR)
-GO_DEVEL_BUILD_BINARY  = $(GO_DEVEL_BUILD_DIR)/commento-ce
+GO_DEVEL_BUILD_BINARY  = $(GO_DEVEL_BUILD_DIR)/commento
 GO_PROD_BUILD_DIR      = $(PROD_BUILD_DIR)
-GO_PROD_BUILD_BINARY   = $(GO_PROD_BUILD_DIR)/commento-ce
+GO_PROD_BUILD_BINARY   = $(GO_PROD_BUILD_DIR)/commento
 
 devel: devel-go
 
@@ -25,15 +25,15 @@ clean:
 # later down the line).
 
 devel-go:
-	go get -v .
+	dep ensure
 	go build -i -v -o $(GO_DEVEL_BUILD_BINARY)
 
 prod-go:
-	go get -v .
+	dep ensure
 	go build -i -v -o $(GO_PROD_BUILD_BINARY)
 
 test-go:
-	go get -v .
+	dep ensure
 	go test -v .
 
 $(shell mkdir -p $(GO_DEVEL_BUILD_DIR) $(GO_PROD_BUILD_DIR))

api/akismet.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"github.com/adtac/go-akismet/akismet"
+	"os"
+)
+
+func isSpam(domain string, userIp string, userAgent string, name string, email string, url string, markdown string) bool {
+	akismetKey := os.Getenv("AKISMET_KEY")
+	if akismetKey == "" {
+		return false
+	}
+
+	res, err := akismet.Check(&akismet.Comment{
+		Blog:               domain,
+		UserIP:             userIp,
+		UserAgent:          userAgent,
+		CommentType:        "comment",
+		CommentAuthor:      name,
+		CommentAuthorEmail: email,
+		CommentAuthorURL:   url,
+		CommentContent:     markdown,
+	}, akismetKey)
+
+	if err != nil {
+		logger.Errorf("error: cannot validate commenet using Akismet: %v", err)
+		return true
+	}
+
+	return res
+}

api/comment.go

@@ -13,7 +13,7 @@ type comment struct {
 	Html         string    `json:"html"`
 	ParentHex    string    `json:"parentHex"`
 	Score        int       `json:"score"`
-	State        string    `json:"state"`
+	State        string    `json:"state,omitempty"`
 	CreationDate time.Time `json:"creationDate"`
 	Direction    int       `json:"direction"`
 }

api/comment_count.go

@@ -1,27 +1,51 @@
 package main
 
 import (
+	"github.com/lib/pq"
 	"net/http"
 )
 
-func commentCount(domain string, path string) (int, error) {
-	// path can be empty
+func commentCount(domain string, paths []string) (map[string]int, error) {
+	commentCounts := map[string]int{}
+
 	if domain == "" {
-		return 0, errorMissingField
+		return nil, errorMissingField
 	}
 
-	p, err := pageGet(domain, path)
+	if len(paths) == 0 {
+		return nil, errorEmptyPaths
+	}
+
+	statement := `
+		SELECT path, commentCount
+		FROM pages
+		WHERE domain = $1 AND path = ANY($2);
+	`
+	rows, err := db.Query(statement, domain, pq.Array(paths))
 	if err != nil {
-		return 0, errorInternal
+		logger.Errorf("cannot get comments: %v", err)
+		return nil, errorInternal
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var path string
+		var commentCount int
+		if err = rows.Scan(&path, &commentCount); err != nil {
+			logger.Errorf("cannot scan path and commentCount: %v", err)
+			return nil, errorInternal
+		}
+
+		commentCounts[path] = commentCount
 	}
 
-	return p.CommentCount, nil
+	return commentCounts, nil
 }
 
 func commentCountHandler(w http.ResponseWriter, r *http.Request) {
 	type request struct {
-		Domain *string `json:"domain"`
-		Path   *string `json:"path"`
+		Domain *string   `json:"domain"`
+		Paths  *[]string `json:"paths"`
 	}
 
 	var x request
@@ -31,13 +55,12 @@ func commentCountHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	domain := domainStrip(*x.Domain)
-	path := *x.Path
 
-	count, err := commentCount(domain, path)
+	commentCounts, err := commentCount(domain, *x.Paths)
 	if err != nil {
 		bodyMarshal(w, response{"success": false, "message": err.Error()})
 		return
 	}
 
-	bodyMarshal(w, response{"success": true, "count": count})
+	bodyMarshal(w, response{"success": true, "commentCounts": commentCounts})
 }

api/comment_count_test.go

@@ -14,14 +14,14 @@ func TestCommentCountBasics(t *testing.T) {
 	commentNew(commenterHex, "example.com", "/path.html", "root", "**bar**", "approved", time.Now().UTC())
 	commentNew(commenterHex, "example.com", "/path.html", "root", "**baz**", "unapproved", time.Now().UTC())
 
-	count, err := commentCount("example.com", "/path.html")
+	counts, err := commentCount("example.com", []string{"/path.html"})
 	if err != nil {
 		t.Errorf("unexpected error counting comments: %v", err)
 		return
 	}
 
-	if count != 2 {
-		t.Errorf("expected count=2 got count=%d", count)
+	if counts["/path.html"] != 2 {
+		t.Errorf("expected count=2 got count=%d", counts["/path.html"])
 		return
 	}
 }
@@ -29,25 +29,25 @@ func TestCommentCountBasics(t *testing.T) {
 func TestCommentCountNewPage(t *testing.T) {
 	failTestOnError(t, setupTestEnv())
 
-	count, err := commentCount("example.com", "/path.html")
+	counts, err := commentCount("example.com", []string{"/path.html"})
 	if err != nil {
 		t.Errorf("unexpected error counting comments: %v", err)
 		return
 	}
 
-	if count != 0 {
-		t.Errorf("expected count=0 got count=%d", count)
+	if counts["/path.html"] != 0 {
+		t.Errorf("expected count=0 got count=%d", counts["/path.html"])
 		return
 	}
 }
 
 func TestCommentCountEmpty(t *testing.T) {
-	if _, err := commentCount("example.com", ""); err != nil {
+	if _, err := commentCount("example.com", []string{""}); err != nil {
 		t.Errorf("unexpected error counting comments on empty path: %v", err)
 		return
 	}
 
-	if _, err := commentCount("", ""); err == nil {
+	if _, err := commentCount("", []string{""}); err == nil {
 		t.Errorf("expected error not found counting comments with empty everything")
 		return
 	}

api/comment_list.go

@@ -12,22 +12,26 @@ func commentList(commenterHex string, domain string, path string, includeUnappro
 	}
 
 	statement := `
-    SELECT commentHex, commenterHex, markdown, html, parentHex, score, state, creationDate
+		SELECT
+			commentHex,
+			commenterHex,
+			markdown,
+			html,
+			parentHex,
+			score,
+			state,
+			creationDate
 		FROM comments
 		WHERE
-      comments.domain = $1 AND
-      comments.path = $2
-  `
+			comments.domain = $1 AND
+			comments.path = $2
+	`
 
 	if !includeUnapproved {
 		if commenterHex == "anonymous" {
-			statement += `
-        AND state = 'approved'
-      `
+			statement += `AND state = 'approved'`
 		} else {
-			statement += `
-        AND (state = 'approved' OR commenterHex = $3)
-      `
+			statement += `AND (state = 'approved' OR commenterHex = $3)`
 		}
 	}
 
@@ -54,16 +58,24 @@ func commentList(commenterHex string, domain string, path string, includeUnappro
 	comments := []comment{}
 	for rows.Next() {
 		c := comment{}
-		if err = rows.Scan(&c.CommentHex, &c.CommenterHex, &c.Markdown, &c.Html, &c.ParentHex, &c.Score, &c.State, &c.CreationDate); err != nil {
+		if err = rows.Scan(
+			&c.CommentHex,
+			&c.CommenterHex,
+			&c.Markdown,
+			&c.Html,
+			&c.ParentHex,
+			&c.Score,
+			&c.State,
+			&c.CreationDate); err != nil {
 			return nil, nil, errorInternal
 		}
 
 		if commenterHex != "anonymous" {
 			statement = `
-        SELECT direction
-        FROM votes
-        WHERE commentHex=$1 AND commenterHex=$2;
-      `
+				SELECT direction
+				FROM votes
+				WHERE commentHex=$1 AND commenterHex=$2;
+			`
 			row := db.QueryRow(statement, c.CommentHex, commenterHex)
 
 			if err = row.Scan(&c.Direction); err != nil {
@@ -120,6 +132,8 @@ func commentListHandler(w http.ResponseWriter, r *http.Request) {
 
 	commenterHex := "anonymous"
 	isModerator := false
+	modList := map[string]bool{}
+
 	if *x.CommenterToken != "anonymous" {
 		c, err := commenterGetByCommenterToken(*x.CommenterToken)
 		if err != nil {
@@ -134,11 +148,15 @@ func commentListHandler(w http.ResponseWriter, r *http.Request) {
 		}
 
 		for _, mod := range d.Moderators {
+			modList[mod.Email] = true
 			if mod.Email == c.Email {
 				isModerator = true
-				break
 			}
 		}
+	} else {
+		for _, mod := range d.Moderators {
+			modList[mod.Email] = true
+		}
 	}
 
 	domainViewRecord(domain, commenterHex)
@@ -149,16 +167,32 @@ func commentListHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	_commenters := map[string]commenter{}
+	for commenterHex, cr := range commenters {
+		if _, ok := modList[cr.Email]; ok {
+			cr.IsModerator = true
+		}
+		cr.Email = ""
+		_commenters[commenterHex] = cr
+	}
+
 	bodyMarshal(w, response{
 		"success":               true,
 		"domain":                domain,
 		"comments":              comments,
-		"commenters":            commenters,
+		"commenters":            _commenters,
 		"requireModeration":     d.RequireModeration,
 		"requireIdentification": d.RequireIdentification,
 		"isFrozen":              d.State == "frozen",
 		"isModerator":           isModerator,
 		"attributes":            p,
-		"configuredOauths":      configuredOauths,
+		"configuredOauths": map[string]bool{
+			"commento": d.CommentoProvider,
+			"google":   googleConfigured && d.GoogleProvider,
+			"twitter":  twitterConfigured && d.TwitterProvider,
+			"github":   githubConfigured && d.GithubProvider,
+			"gitlab":   gitlabConfigured && d.GitlabProvider,
+			"sso":      d.SsoProvider,
+		},
 	})
 }

api/comment_new.go

@@ -77,20 +77,34 @@ func commentNewHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if d.RequireIdentification && *x.CommenterToken == "anonymous" {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
 	// logic: (empty column indicates the value doesn't matter)
-	// | anonymous | moderator | requireIdentification | requireModeration | approved? |
-	// |-----------+-----------+-----------------------+-------------------+-----------|
-	// |       yes |           |                       |                   |        no |
-	// |        no |       yes |                       |                   |       yes |
-	// |        no |        no |                       |               yes |       yes |
-	// |        no |        no |                       |                no |        no |
+	// | anonymous | moderator | requireIdentification | requireModeration | moderateAllAnonymous | approved? |
+	// |-----------+-----------+-----------------------+-------------------+----------------------+-----------|
+	// |       yes |           |                       |                   |                   no |       yes |
+	// |       yes |           |                       |                   |                  yes |        no |
+	// |        no |       yes |                       |                   |                      |       yes |
+	// |        no |        no |                       |               yes |                      |       yes |
+	// |        no |        no |                       |                no |                      |        no |
 
 	var commenterHex string
 	var state string
 
 	if *x.CommenterToken == "anonymous" {
-		state = "unapproved"
 		commenterHex = "anonymous"
+		if isSpam(*x.Domain, getIp(r), getUserAgent(r), "Anonymous", "", "", *x.Markdown) {
+			state = "flagged"
+		} else {
+			if d.ModerateAllAnonymous || d.RequireModeration {
+				state = "unapproved"
+			} else {
+				state = "approved"
+			}
+		}
 	} else {
 		c, err := commenterGetByCommenterToken(*x.CommenterToken)
 		if err != nil {
@@ -112,10 +126,14 @@ func commentNewHandler(w http.ResponseWriter, r *http.Request) {
 		if isModerator {
 			state = "approved"
 		} else {
-			if d.RequireModeration {
-				state = "unapproved"
+			if isSpam(*x.Domain, getIp(r), getUserAgent(r), c.Name, c.Email, c.Link, *x.Markdown) {
+				state = "flagged"
 			} else {
-				state = "approved"
+				if d.RequireModeration {
+					state = "unapproved"
+				} else {
+					state = "approved"
+				}
 			}
 		}
 	}
@@ -126,5 +144,11 @@ func commentNewHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bodyMarshal(w, response{"success": true, "commentHex": commentHex, "approved": state == "approved"})
+	// TODO: reuse html in commentNew and do only one markdown to HTML conversion?
+	html := markdownToHtml(*x.Markdown)
+
+	bodyMarshal(w, response{"success": true, "commentHex": commentHex, "state": state, "html": html})
+	if smtpConfigured {
+		go emailNotificationNew(d, path, commenterHex, commentHex, *x.ParentHex, state)
+	}
 }

api/commenter.go

@@ -12,4 +12,5 @@ type commenter struct {
 	Photo        string    `json:"photo"`
 	Provider     string    `json:"provider,omitempty"`
 	JoinDate     time.Time `json:"joinDate,omitempty"`
+	IsModerator  bool      `json:"isModerator"`
 }

api/commenter_get.go

@@ -66,5 +66,6 @@ func commenterGetByCommenterToken(commenterToken string) (commenter, error) {
 		return commenter{}, errorNoSuchToken
 	}
 
+	// TODO: use a join instead of two queries?
 	return commenterGetByHex(commenterHex)
 }

api/commenter_login.go

@@ -67,5 +67,12 @@ func commenterLoginHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bodyMarshal(w, response{"success": true, "commenterToken": commenterToken})
+	// TODO: modify commenterLogin to directly return c?
+	c, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true, "commenterToken": commenterToken, "commenter": c})
 }

api/commenter_new.go

@@ -27,6 +27,10 @@ func commenterNew(email string, name string, link string, photo string, provider
 		return "", errorEmailAlreadyExists
 	}
 
+	if err := emailNew(email); err != nil {
+		return "", errorInternal
+	}
+
 	commenterHex, err := randomHex(32)
 	if err != nil {
 		return "", errorInternal

api/commenter_photo.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"io"
+	"net/http"
+)
+
+func commenterPhotoHandler(w http.ResponseWriter, r *http.Request) {
+	c, err := commenterGetByHex(r.FormValue("commenterHex"))
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	url := c.Photo
+	if c.Provider == "google" {
+		url += "?sz=50"
+	} else if c.Provider == "github" {
+		url += "&s=50"
+	} else if c.Provider == "twitter" {
+		url += "?size=normal"
+	} else if c.Provider == "gitlab" {
+		url += "?width=50"
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+	defer resp.Body.Close()
+
+	io.Copy(w, resp.Body)
+}

api/commenter_self.go

@@ -21,5 +21,11 @@ func commenterSelfHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bodyMarshal(w, response{"success": true, "commenter": c})
+	e, err := emailGet(c.Email)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true, "commenter": c, "email": e})
 }

api/config.go

@@ -43,8 +43,19 @@ func configParse() error {
 		"SMTP_PORT":         "",
 		"SMTP_FROM_ADDRESS": "",
 
+		"AKISMET_KEY": "",
+
 		"GOOGLE_KEY":    "",
 		"GOOGLE_SECRET": "",
+
+		"GITHUB_KEY":    "",
+		"GITHUB_SECRET": "",
+
+		"TWITTER_KEY":    "",
+		"TWITTER_SECRET": "",
+
+		"GITLAB_KEY":    "",
+		"GITLAB_SECRET": "",
 	}
 
 	for key, value := range defaults {

api/constants.go

@@ -1,4 +1,3 @@
 package main
 
-var edition = "ce"
-var version = "v1.1.0"
+var version = "v1.7.0"

api/cron_domain_export_cleanup.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"time"
+)
+
+func domainExportCleanupBegin() error {
+	go func() {
+		for {
+			statement := `
+				DELETE FROM exports
+				WHERE creationDate < $1;
+			`
+			_, err := db.Exec(statement, time.Now().UTC().AddDate(0, 0, -7))
+			if err != nil {
+				logger.Errorf("error cleaning up export rows: %v", err)
+				return
+			}
+
+			time.Sleep(2 * time.Hour)
+		}
+	}()
+
+	return nil
+}

api/cron_email_notification.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"time"
+)
+
+func emailNotificationBegin() error {
+	go func() {
+		for {
+			statement := `
+				SELECT email, sendModeratorNotifications, sendReplyNotifications
+				FROM emails
+				WHERE pendingEmails > 0 AND lastEmailNotificationDate < $1;
+			`
+			rows, err := db.Query(statement, time.Now().UTC().Add(time.Duration(-10)*time.Minute))
+			if err != nil {
+				logger.Errorf("cannot query domains: %v", err)
+				return
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				var email string
+				var sendModeratorNotifications bool
+				var sendReplyNotifications bool
+				if err = rows.Scan(&email, &sendModeratorNotifications, &sendReplyNotifications); err != nil {
+					logger.Errorf("cannot scan email in cron job to send notifications: %v", err)
+					continue
+				}
+
+				if _, ok := emailQueue[email]; !ok {
+					if err = emailNotificationPendingReset(email); err != nil {
+						logger.Errorf("error resetting pendingEmails: %v", err)
+						continue
+					}
+				}
+
+				cont := true
+				kindListMap := map[string][]emailNotification{}
+				for cont {
+					select {
+					case e := <-emailQueue[email]:
+						if _, ok := kindListMap[e.Kind]; !ok {
+							kindListMap[e.Kind] = []emailNotification{}
+						}
+
+						if (e.Kind == "reply" && sendReplyNotifications) || sendModeratorNotifications {
+							kindListMap[e.Kind] = append(kindListMap[e.Kind], e)
+						}
+					default:
+						cont = false
+						break
+					}
+				}
+
+				for kind, list := range kindListMap {
+					go emailNotificationSend(email, kind, list)
+				}
+			}
+
+			time.Sleep(10 * time.Minute)
+		}
+	}()
+
+	return nil
+}

api/cron_sso_token.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"time"
+)
+
+func ssoTokenCleanupBegin() error {
+	go func() {
+		for {
+			statement := `
+				DELETE FROM ssoTokens
+				WHERE creationDate < $1;
+			`
+			_, err := db.Exec(statement, time.Now().UTC().Add(time.Duration(-10)*time.Minute))
+			if err != nil {
+				logger.Errorf("error cleaning up export rows: %v", err)
+				return
+			}
+
+			time.Sleep(10 * time.Minute)
+		}
+	}()
+
+	return nil
+}

api/cron_views_cleanup.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"time"
+)
+
+func viewsCleanupBegin() error {
+	go func() {
+		for {
+			statement := `
+				DELETE FROM views
+				WHERE viewDate < $1;
+			`
+			_, err := db.Exec(statement, time.Now().UTC().AddDate(0, 0, -45))
+			if err != nil {
+				logger.Errorf("error cleaning up views: %v", err)
+				return
+			}
+
+			time.Sleep(24 * time.Hour)
+		}
+	}()
+
+	return nil
+}

api/database_migrate.go

@@ -6,6 +6,10 @@ import (
 	"strings"
 )
 
+var goMigrations = map[string](func() error){
+	"20190213033530-email-notifications.sql": migrateEmails,
+}
+
 func migrate() error {
 	return migrateFromDir(os.Getenv("STATIC") + "/db")
 }
@@ -69,6 +73,13 @@ func migrateFromDir(dir string) error {
 					return err
 				}
 
+				if fn, ok := goMigrations[file.Name()]; ok {
+					if err = fn(); err != nil {
+						logger.Errorf("cannot execute Go migration associated with SQL %s: %v", f, err)
+						return err
+					}
+				}
+
 				completed++
 			}
 		}

api/database_migrate_email_notifications.go

@@ -0,0 +1,37 @@
+package main
+
+import ()
+
+func migrateEmails() error {
+	statement := `
+		SELECT commenters.email
+		FROM commenters
+		UNION
+		SELECT owners.email
+		FROM owners
+		UNION
+		SELECT moderators.email
+		FROM moderators;
+	`
+	rows, err := db.Query(statement)
+	if err != nil {
+		logger.Errorf("cannot get comments: %v", err)
+		return errorDatabaseMigration
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var email string
+		if err = rows.Scan(&email); err != nil {
+			logger.Errorf("cannot get email from tables during migration: %v", err)
+			return errorDatabaseMigration
+		}
+
+		if err = emailNew(email); err != nil {
+			logger.Errorf("cannot insert email during migration: %v", err)
+			return errorDatabaseMigration
+		}
+	}
+
+	return nil
+}

api/domain.go

@@ -5,14 +5,24 @@ import (
 )
 
 type domain struct {
-	Domain                string      `json:"domain"`
-	OwnerHex              string      `json:"ownerHex"`
-	Name                  string      `json:"name"`
-	CreationDate          time.Time   `json:"creationDate"`
-	State                 string      `json:"state"`
-	ImportedComments      bool        `json:"importedComments"`
-	AutoSpamFilter        bool        `json:"autoSpamFilter"`
-	RequireModeration     bool        `json:"requireModeration"`
-	RequireIdentification bool        `json:"requireIdentification"`
-	Moderators            []moderator `json:"moderators"`
+	Domain                  string      `json:"domain"`
+	OwnerHex                string      `json:"ownerHex"`
+	Name                    string      `json:"name"`
+	CreationDate            time.Time   `json:"creationDate"`
+	State                   string      `json:"state"`
+	ImportedComments        bool        `json:"importedComments"`
+	AutoSpamFilter          bool        `json:"autoSpamFilter"`
+	RequireModeration       bool        `json:"requireModeration"`
+	RequireIdentification   bool        `json:"requireIdentification"`
+	ModerateAllAnonymous    bool        `json:"moderateAllAnonymous"`
+	Moderators              []moderator `json:"moderators"`
+	EmailNotificationPolicy string      `json:"emailNotificationPolicy"`
+	CommentoProvider        bool        `json:"commentoProvider"`
+	GoogleProvider          bool        `json:"googleProvider"`
+	TwitterProvider         bool        `json:"twitterProvider"`
+	GithubProvider          bool        `json:"githubProvider"`
+	GitlabProvider          bool        `json:"gitlabProvider"`
+	SsoProvider             bool        `json:"ssoProvider"`
+	SsoSecret               string      `json:"ssoSecret"`
+	SsoUrl                  string      `json:"ssoUrl"`
 }

api/domain_clear.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"net/http"
+)
+
+func domainClear(domain string) error {
+	if domain == "" {
+		return errorMissingField
+	}
+
+	statement := `
+		DELETE FROM votes
+		USING comments
+		WHERE comments.commentHex = votes.commentHex AND comments.domain = $1;
+	`
+	_, err := db.Exec(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot delete votes: %v", err)
+		return errorInternal
+	}
+
+	statement = `
+		DELETE FROM comments
+		WHERE comments.domain = $1;
+	`
+	_, err = db.Exec(statement, domain)
+	if err != nil {
+		logger.Errorf(statement, domain)
+		return errorInternal
+	}
+
+	statement = `
+		DELETE FROM pages
+		WHERE pages.domain = $1;
+	`
+	_, err = db.Exec(statement, domain)
+	if err != nil {
+		logger.Errorf(statement, domain)
+		return errorInternal
+	}
+
+	return nil
+}
+
+func domainClearHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain     *string `json:"domain"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	domain := domainStrip(*x.Domain)
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	if err = domainClear(*x.Domain); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true})
+}

api/domain_delete.go

@@ -19,17 +19,6 @@ func domainDelete(domain string) error {
 		return errorNoSuchDomain
 	}
 
-	statement = `
-		DELETE FROM votes
-		USING comments
-		WHERE comments.commentHex = votes.commentHex AND comments.domain = $1;
-	`
-	_, err = db.Exec(statement, domain)
-	if err != nil {
-		logger.Errorf("cannot delete votes: %v", err)
-		return errorInternal
-	}
-
 	statement = `
 		DELETE FROM views
 		WHERE views.domain = $1;
@@ -50,13 +39,9 @@ func domainDelete(domain string) error {
 		return errorInternal
 	}
 
-	statement = `
-		DELETE FROM comments
-		WHERE comments.domain = $1;
-	`
-	_, err = db.Exec(statement, domain)
-	if err != nil {
-		logger.Errorf(statement, domain)
+	// comments, votes, and pages are handled by domainClear
+	if err = domainClear(domain); err != nil {
+		logger.Errorf("cannot clear domain: %v", err)
 		return errorInternal
 	}
 

api/domain_export.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+func domainExportBeginError(email string, toName string, domain string, err error) {
+	// we're not using err at the moment because it's all errorInternal
+	if err2 := smtpDomainExportError(email, toName, domain); err2 != nil {
+		logger.Errorf("cannot send domain export error email for %s: %v", domain, err2)
+		return
+	}
+}
+
+func domainExportBegin(email string, toName string, domain string) {
+	type dataExport struct {
+		Version    int         `json:"version"`
+		Comments   []comment   `json:"comments"`
+		Commenters []commenter `json:"commenters"`
+	}
+
+	e := dataExport{Version: 1, Comments: []comment{}, Commenters: []commenter{}}
+
+	statement := `
+		SELECT commentHex, domain, path, commenterHex, markdown, parentHex, score, state, creationDate
+		FROM comments
+		WHERE domain = $1;
+	`
+	rows1, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select comments while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows1.Close()
+
+	for rows1.Next() {
+		c := comment{}
+		if err = rows1.Scan(&c.CommentHex, &c.Domain, &c.Path, &c.CommenterHex, &c.Markdown, &c.ParentHex, &c.Score, &c.State, &c.CreationDate); err != nil {
+			logger.Errorf("cannot scan comment while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Comments = append(e.Comments, c)
+	}
+
+	statement = `
+		SELECT commenters.commenterHex, commenters.email, commenters.name, commenters.link, commenters.photo, commenters.provider, commenters.joinDate
+		FROM commenters, comments
+		WHERE comments.domain = $1 AND commenters.commenterHex = comments.commenterHex;
+	`
+	rows2, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select commenters while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows2.Close()
+
+	for rows2.Next() {
+		c := commenter{}
+		if err := rows2.Scan(&c.CommenterHex, &c.Email, &c.Name, &c.Link, &c.Photo, &c.Provider, &c.JoinDate); err != nil {
+			logger.Errorf("cannot scan commenter while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Commenters = append(e.Commenters, c)
+	}
+
+	je, err := json.Marshal(e)
+	if err != nil {
+		logger.Errorf("cannot marshall JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	gje, err := gzipStatic(je)
+	if err != nil {
+		logger.Errorf("cannot gzip JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	exportHex, err := randomHex(32)
+	if err != nil {
+		logger.Errorf("cannot generate exportHex while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	statement = `
+		INSERT INTO
+		exports (exportHex, binData, domain, creationDate)
+		VALUES  ($1,        $2,      $3    , $4          );
+	`
+	_, err = db.Exec(statement, exportHex, gje, domain, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("error inserting expiry binary data while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	err = smtpDomainExport(email, toName, domain, exportHex)
+	if err != nil {
+		logger.Errorf("error sending data export email for %s: %v", domain, err)
+		return
+	}
+}
+
+func domainExportBeginHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain     *string `json:"domain"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !smtpConfigured {
+		bodyMarshal(w, response{"success": false, "message": errorSmtpNotConfigured.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, *x.Domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	go domainExportBegin(o.Email, o.Name, *x.Domain)
+
+	bodyMarshal(w, response{"success": true})
+}

api/domain_export_download.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func domainExportDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	exportHex := r.FormValue("exportHex")
+	if exportHex == "" {
+		fmt.Fprintf(w, "Error: empty exportHex\n")
+		return
+	}
+
+	statement := `
+		SELECT domain, binData, creationDate
+		FROM exports
+		WHERE exportHex = $1;
+	`
+	row := db.QueryRow(statement, exportHex)
+
+	var domain string
+	var binData []byte
+	var creationDate time.Time
+	if err := row.Scan(&domain, &binData, &creationDate); err != nil {
+		fmt.Fprintf(w, "Error: that exportHex does not exist\n")
+	}
+
+	w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s-%v.gz"`, domain, creationDate.Unix()))
+	w.Header().Set("Content-Encoding", "gzip")
+	w.Write(binData)
+}

api/domain_get.go

@@ -8,7 +8,26 @@ func domainGet(dmn string) (domain, error) {
 	}
 
 	statement := `
-    SELECT domain, ownerHex, name, creationDate, state, importedComments, autoSpamFilter, requireModeration, requireIdentification
+		SELECT
+			domain,
+			ownerHex,
+			name,
+			creationDate,
+			state,
+			importedComments,
+			autoSpamFilter,
+			requireModeration,
+			requireIdentification,
+			moderateAllAnonymous,
+			emailNotificationPolicy,
+			commentoProvider,
+			googleProvider,
+			twitterProvider,
+			githubProvider,
+			gitlabProvider,
+			ssoProvider,
+			ssoSecret,
+			ssoUrl
 		FROM domains
 		WHERE domain = $1;
 	`
@@ -16,7 +35,26 @@ func domainGet(dmn string) (domain, error) {
 
 	var err error
 	d := domain{}
-	if err = row.Scan(&d.Domain, &d.OwnerHex, &d.Name, &d.CreationDate, &d.State, &d.ImportedComments, &d.AutoSpamFilter, &d.RequireModeration, &d.RequireIdentification); err != nil {
+	if err = row.Scan(
+		&d.Domain,
+		&d.OwnerHex,
+		&d.Name,
+		&d.CreationDate,
+		&d.State,
+		&d.ImportedComments,
+		&d.AutoSpamFilter,
+		&d.RequireModeration,
+		&d.RequireIdentification,
+		&d.ModerateAllAnonymous,
+		&d.EmailNotificationPolicy,
+		&d.CommentoProvider,
+		&d.GoogleProvider,
+		&d.TwitterProvider,
+		&d.GithubProvider,
+		&d.GitlabProvider,
+		&d.SsoProvider,
+		&d.SsoSecret,
+		&d.SsoUrl); err != nil {
 		return d, errorNoSuchDomain
 	}
 

api/domain_import_disqus.go

@@ -18,9 +18,9 @@ type disqusThread struct {
 
 type disqusAuthor struct {
 	XMLName     xml.Name `xml:"author"`
-	IsAnonymous bool     `xml:"isAnonymous"`
 	Name        string   `xml:"name"`
-	Email       string   `xml:"email"`
+	IsAnonymous bool     `xml:"isAnonymous"`
+	Username    string   `xml:"username"`
 }
 
 type disqusThreadId struct {
@@ -43,7 +43,6 @@ type disqusPost struct {
 	Id           string         `xml:"http://disqus.com/disqus-internals id,attr"`
 	ThreadId     disqusThreadId `xml:"thread"`
 	ParentId     disqusParentId `xml:"parent"`
-	PostId       disqusPostId   `xml:"post"`
 	Message      string         `xml:"message"`
 	CreationDate time.Time      `xml:"createdAt"`
 	IsDeleted    bool           `xml:"isDeleted"`
@@ -98,24 +97,26 @@ func domainImportDisqus(domain string, url string) (int, error) {
 
 	// Map Disqus emails to commenterHex (if not available, create a new one
 	// with a random password that can be reset later).
-	commenterHex := make(map[string]string)
+	commenterHex := map[string]string{}
 	for _, post := range x.Posts {
 		if post.IsDeleted || post.IsSpam {
 			continue
 		}
 
-		if _, ok := commenterHex[post.Author.Email]; ok {
+		email := post.Author.Username + "@disqus.com"
+
+		if _, ok := commenterHex[email]; ok {
 			continue
 		}
 
-		c, err := commenterGetByEmail("commento", post.Author.Email)
+		c, err := commenterGetByEmail("commento", email)
 		if err != nil && err != errorNoSuchCommenter {
 			logger.Errorf("cannot get commenter by email: %v", err)
 			return 0, errorInternal
 		}
 
 		if err == nil {
-			commenterHex[post.Author.Email] = c.CommenterHex
+			commenterHex[email] = c.CommenterHex
 			continue
 		}
 
@@ -125,7 +126,7 @@ func domainImportDisqus(domain string, url string) (int, error) {
 			return 0, errorInternal
 		}
 
-		commenterHex[post.Author.Email], err = commenterNew(post.Author.Email, post.Author.Name, "undefined", "undefined", "commento", randomPassword)
+		commenterHex[email], err = commenterNew(email, post.Author.Name, "undefined", "undefined", "commento", randomPassword)
 		if err != nil {
 			return 0, err
 		}
@@ -134,12 +135,17 @@ func domainImportDisqus(domain string, url string) (int, error) {
 	// For each Disqus post, create a Commento comment. Attempt to convert the
 	// HTML to markdown.
 	numImported := 0
-	disqusIdMap := make(map[string]string)
+	disqusIdMap := map[string]string{}
 	for _, post := range x.Posts {
 		if post.IsDeleted || post.IsSpam {
 			continue
 		}
 
+		cHex := "anonymous"
+		if !post.Author.IsAnonymous {
+			cHex = commenterHex[post.Author.Username+"@disqus.com"]
+		}
+
 		parentHex := "root"
 		if val, ok := disqusIdMap[post.ParentId.Id]; ok {
 			parentHex = val
@@ -148,7 +154,7 @@ func domainImportDisqus(domain string, url string) (int, error) {
 		// TODO: restrict the list of tags to just the basics: <a>, <b>, <i>, <code>
 		// Especially remove <img> (convert it to <a>).
 		commentHex, err := commentNew(
-			commenterHex[post.Author.Email],
+			cHex,
 			domain,
 			pathStrip(threads[post.ThreadId.Id].URL),
 			parentHex,
@@ -159,7 +165,7 @@ func domainImportDisqus(domain string, url string) (int, error) {
 			return numImported, err
 		}
 
-		disqusIdMap[post.PostId.Id] = commentHex
+		disqusIdMap[post.Id] = commentHex
 		numImported += 1
 	}
 

api/domain_list.go

@@ -10,7 +10,26 @@ func domainList(ownerHex string) ([]domain, error) {
 	}
 
 	statement := `
-    SELECT domain, ownerHex, name, creationDate, state, importedComments, autoSpamFilter, requireModeration, requireIdentification
+		SELECT
+			domain,
+			ownerHex,
+			name,
+			creationDate,
+			state,
+			importedComments,
+			autoSpamFilter,
+			requireModeration,
+			requireIdentification,
+			moderateAllAnonymous,
+			emailNotificationPolicy,
+			commentoProvider,
+			googleProvider,
+			twitterProvider,
+			githubProvider,
+			gitlabProvider,
+			ssoProvider,
+			ssoSecret,
+			ssoUrl
 		FROM domains
 		WHERE ownerHex=$1;
 	`
@@ -24,7 +43,26 @@ func domainList(ownerHex string) ([]domain, error) {
 	domains := []domain{}
 	for rows.Next() {
 		d := domain{}
-		if err = rows.Scan(&d.Domain, &d.OwnerHex, &d.Name, &d.CreationDate, &d.State, &d.ImportedComments, &d.AutoSpamFilter, &d.RequireModeration, &d.RequireIdentification); err != nil {
+		if err = rows.Scan(
+			&d.Domain,
+			&d.OwnerHex,
+			&d.Name,
+			&d.CreationDate,
+			&d.State,
+			&d.ImportedComments,
+			&d.AutoSpamFilter,
+			&d.RequireModeration,
+			&d.RequireIdentification,
+			&d.ModerateAllAnonymous,
+			&d.EmailNotificationPolicy,
+			&d.CommentoProvider,
+			&d.GoogleProvider,
+			&d.TwitterProvider,
+			&d.GithubProvider,
+			&d.GitlabProvider,
+			&d.SsoProvider,
+			&d.SsoSecret,
+			&d.SsoUrl); err != nil {
 			logger.Errorf("cannot Scan domain: %v", err)
 			return nil, errorInternal
 		}
@@ -63,5 +101,14 @@ func domainListHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bodyMarshal(w, response{"success": true, "domains": domains})
+	bodyMarshal(w, response{
+		"success": true,
+		"domains": domains,
+		"configuredOauths": map[string]bool{
+			"google":  googleConfigured,
+			"twitter": twitterConfigured,
+			"github":  githubConfigured,
+			"gitlab":  gitlabConfigured,
+		},
+	})
 }

api/domain_moderator_new.go

@@ -10,6 +10,11 @@ func domainModeratorNew(domain string, email string) error {
 		return errorMissingField
 	}
 
+	if err := emailNew(email); err != nil {
+		logger.Errorf("cannot create email when creating moderator: %v", err)
+		return errorInternal
+	}
+
 	statement := `
 		INSERT INTO
 		moderators (domain, email, addDate)

api/domain_new.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"net/http"
+	"strings"
 	"time"
 )
 
@@ -10,6 +11,10 @@ func domainNew(ownerHex string, name string, domain string) error {
 		return errorMissingField
 	}
 
+	if strings.Contains(domain, "/") {
+		return errorInvalidDomain
+	}
+
 	statement := `
 		INSERT INTO
 		domains (ownerHex, name, domain, creationDate)

api/domain_sso.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"net/http"
+)
+
+func domainSsoSecretNew(domain string) (string, error) {
+	if domain == "" {
+		return "", errorMissingField
+	}
+
+	ssoSecret, err := randomHex(32)
+	if err != nil {
+		logger.Errorf("error generating SSO secret hex: %v", err)
+		return "", errorInternal
+	}
+
+	statement := `
+		UPDATE domains
+		SET ssoSecret = $2
+		WHERE domain = $1;
+	`
+	_, err = db.Exec(statement, domain, ssoSecret)
+	if err != nil {
+		logger.Errorf("cannot update ssoSecret: %v", err)
+		return "", errorInternal
+	}
+
+	return ssoSecret, nil
+}
+
+func domainSsoSecretNewHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain     *string `json:"domain"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	domain := domainStrip(*x.Domain)
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	ssoSecret, err := domainSsoSecretNew(domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true, "ssoSecret": ssoSecret})
+}

api/domain_update.go

@@ -5,13 +5,46 @@ import (
 )
 
 func domainUpdate(d domain) error {
+	if d.SsoProvider && d.SsoUrl == "" {
+		return errorMissingField
+	}
+
 	statement := `
 		UPDATE domains
-    SET name=$2, state=$3, autoSpamFilter=$4, requireModeration=$5, requireIdentification=$6
+		SET
+			name=$2,
+			state=$3,
+			autoSpamFilter=$4,
+			requireModeration=$5,
+			requireIdentification=$6,
+			moderateAllAnonymous=$7,
+			emailNotificationPolicy=$8,
+			commentoProvider=$9,
+			googleProvider=$10,
+			twitterProvider=$11,
+			githubProvider=$12,
+			gitlabProvider=$13,
+			ssoProvider=$14,
+			ssoUrl=$15
 		WHERE domain=$1;
 	`
 
-	_, err := db.Exec(statement, d.Domain, d.Name, d.State, d.AutoSpamFilter, d.RequireModeration, d.RequireIdentification)
+	_, err := db.Exec(statement,
+		d.Domain,
+		d.Name,
+		d.State,
+		d.AutoSpamFilter,
+		d.RequireModeration,
+		d.RequireIdentification,
+		d.ModerateAllAnonymous,
+		d.EmailNotificationPolicy,
+		d.CommentoProvider,
+		d.GoogleProvider,
+		d.TwitterProvider,
+		d.GithubProvider,
+		d.GitlabProvider,
+		d.SsoProvider,
+		d.SsoUrl)
 	if err != nil {
 		logger.Errorf("cannot update non-moderators: %v", err)
 		return errorInternal

api/email.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"time"
+)
+
+type email struct {
+	Email                      string    `json:"email"`
+	UnsubscribeSecretHex       string    `json:"unsubscribeSecretHex"`
+	LastEmailNotificationDate  time.Time `json:"lastEmailNotificationDate"`
+	PendingEmails              int       `json:"-"`
+	SendReplyNotifications     bool      `json:"sendReplyNotifications"`
+	SendModeratorNotifications bool      `json:"sendModeratorNotifications"`
+}

api/email_get.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"net/http"
+)
+
+func emailGet(em string) (email, error) {
+	statement := `
+		SELECT email, unsubscribeSecretHex, lastEmailNotificationDate, pendingEmails, sendReplyNotifications, sendModeratorNotifications
+		FROM emails
+		WHERE email = $1;
+	`
+	row := db.QueryRow(statement, em)
+
+	e := email{}
+	if err := row.Scan(&e.Email, &e.UnsubscribeSecretHex, &e.LastEmailNotificationDate, &e.PendingEmails, &e.SendReplyNotifications, &e.SendModeratorNotifications); err != nil {
+		// TODO: is this the only error?
+		return e, errorNoSuchEmail
+	}
+
+	return e, nil
+}
+
+func emailGetByUnsubscribeSecretHex(unsubscribeSecretHex string) (email, error) {
+	statement := `
+		SELECT email, unsubscribeSecretHex, lastEmailNotificationDate, pendingEmails, sendReplyNotifications, sendModeratorNotifications
+		FROM emails
+		WHERE unsubscribeSecretHex = $1;
+	`
+	row := db.QueryRow(statement, unsubscribeSecretHex)
+
+	e := email{}
+	if err := row.Scan(&e.Email, &e.UnsubscribeSecretHex, &e.LastEmailNotificationDate, &e.PendingEmails, &e.SendReplyNotifications, &e.SendModeratorNotifications); err != nil {
+		// TODO: is this the only error?
+		return e, errorNoSuchUnsubscribeSecretHex
+	}
+
+	return e, nil
+}
+
+func emailGetHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		UnsubscribeSecretHex *string `json:"unsubscribeSecretHex"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	e, err := emailGetByUnsubscribeSecretHex(*x.UnsubscribeSecretHex)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true, "email": e})
+}

api/email_moderate.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func emailModerateHandler(w http.ResponseWriter, r *http.Request) {
+	unsubscribeSecretHex := r.FormValue("unsubscribeSecretHex")
+	e, err := emailGetByUnsubscribeSecretHex(unsubscribeSecretHex)
+	if err != nil {
+		fmt.Fprintf(w, "error: %v", err.Error())
+		return
+	}
+
+	action := r.FormValue("action")
+	if action != "delete" && action != "approve" {
+		fmt.Fprintf(w, "error: invalid action")
+		return
+	}
+
+	commentHex := r.FormValue("commentHex")
+	if commentHex == "" {
+		fmt.Fprintf(w, "error: invalid commentHex")
+		return
+	}
+
+	statement := `
+		SELECT domain
+		FROM comments
+		WHERE commentHex = $1;
+	`
+	row := db.QueryRow(statement, commentHex)
+
+	var domain string
+	if err = row.Scan(&domain); err != nil {
+		// TODO: is this the only error?
+		fmt.Fprintf(w, "error: no such comment found (perhaps it has been deleted?)")
+		return
+	}
+
+	isModerator, err := isDomainModerator(domain, e.Email)
+	if err != nil {
+		logger.Errorf("error checking if %s is a moderator: %v", e.Email, err)
+		fmt.Fprintf(w, "error checking if %s is a moderator: %v", e.Email, err)
+		return
+	}
+
+	if !isModerator {
+		fmt.Fprintf(w, "error: you're not a moderator for that domain")
+		return
+	}
+
+	if action == "approve" {
+		err = commentApprove(commentHex)
+	} else {
+		err = commentDelete(commentHex)
+	}
+
+	if err != nil {
+		fmt.Fprintf(w, "error: %v", err)
+		return
+	}
+
+	fmt.Fprintf(w, "comment successfully %sd", action)
+}

api/email_new.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"time"
+)
+
+func emailNew(email string) error {
+	unsubscribeSecretHex, err := randomHex(32)
+	if err != nil {
+		return errorInternal
+	}
+
+	statement := `
+		INSERT INTO
+		emails (email, unsubscribeSecretHex, lastEmailNotificationDate)
+		VALUES ($1,    $2,                   $3                       )
+		ON CONFLICT DO NOTHING;
+	`
+	_, err = db.Exec(statement, email, unsubscribeSecretHex, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("cannot insert email into emails: %v", err)
+		return errorInternal
+	}
+
+	return nil
+}

api/email_notification.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"time"
+)
+
+type emailNotification struct {
+	Email         string
+	CommenterName string
+	Domain        string
+	Path          string
+	Title         string
+	CommentHex    string
+	Kind          string
+}
+
+var emailQueue map[string](chan emailNotification) = map[string](chan emailNotification){}
+
+func emailNotificationPendingResetAll() error {
+	statement := `
+		UPDATE emails
+		SET pendingEmails = 0;
+	`
+	_, err := db.Exec(statement)
+	if err != nil {
+		logger.Errorf("cannot reset pendingEmails: %v", err)
+		return err
+	}
+
+	return nil
+}
+
+func emailNotificationPendingIncrement(email string) error {
+	statement := `
+		UPDATE emails
+		SET pendingEmails = pendingEmails + 1
+		WHERE email = $1;
+	`
+	_, err := db.Exec(statement, email)
+	if err != nil {
+		logger.Errorf("cannot increment pendingEmails: %v", err)
+		return err
+	}
+
+	return nil
+}
+
+func emailNotificationPendingReset(email string) error {
+	statement := `
+		UPDATE emails
+		SET pendingEmails = 0, lastEmailNotificationDate = $2
+		WHERE email = $1;
+	`
+	_, err := db.Exec(statement, email, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("cannot decrement pendingEmails: %v", err)
+		return err
+	}
+
+	return nil
+}
+
+func emailNotificationEnqueue(e emailNotification) error {
+	if err := emailNotificationPendingIncrement(e.Email); err != nil {
+		logger.Errorf("cannot increment pendingEmails when enqueueing: %v", err)
+		return err
+	}
+
+	if _, ok := emailQueue[e.Email]; !ok {
+		// don't enqueue more than 10 emails as we won't send more than 10 comments
+		// in one email anyway
+		emailQueue[e.Email] = make(chan emailNotification, 10)
+	}
+
+	select {
+	case emailQueue[e.Email] <- e:
+	default:
+	}
+
+	return nil
+}

api/email_notification_new.go

@@ -0,0 +1,138 @@
+package main
+
+import ()
+
+func emailNotificationModerator(d domain, path string, title string, commenterHex string, commentHex string, state string) {
+	if d.EmailNotificationPolicy == "none" {
+		return
+	}
+
+	// We'll need to check again when we're sending in case the comment was
+	// approved midway anyway.
+	if d.EmailNotificationPolicy == "pending-moderation" && state == "approved" {
+		return
+	}
+
+	var commenterName string
+	var commenterEmail string
+	if commenterHex == "anonymous" {
+		commenterName = "Anonymous"
+	} else {
+		c, err := commenterGetByHex(commenterHex)
+		if err != nil {
+			logger.Errorf("cannot get commenter to send email notification: %v", err)
+			return
+		}
+
+		commenterName = c.Name
+		commenterEmail = c.Email
+	}
+
+	kind := d.EmailNotificationPolicy
+	if state != "approved" {
+		kind = "pending-moderation"
+	}
+
+	for _, m := range d.Moderators {
+		// Do not email the commenting moderator their own comment.
+		if commenterHex != "anonymous" && m.Email == commenterEmail {
+			continue
+		}
+
+		emailNotificationPendingIncrement(m.Email)
+		emailNotificationEnqueue(emailNotification{
+			Email:         m.Email,
+			CommenterName: commenterName,
+			Domain:        d.Domain,
+			Path:          path,
+			Title:         title,
+			CommentHex:    commentHex,
+			Kind:          kind,
+		})
+	}
+}
+
+func emailNotificationReply(d domain, path string, title string, commenterHex string, commentHex string, parentHex string, state string) {
+	// No reply notifications for root comments.
+	if parentHex == "root" {
+		return
+	}
+
+	// No reply notification emails for unapproved comments.
+	if state != "approved" {
+		return
+	}
+
+	statement := `
+		SELECT commenterHex
+		FROM comments
+		WHERE commentHex = $1;
+	`
+	row := db.QueryRow(statement, parentHex)
+
+	var parentCommenterHex string
+	err := row.Scan(&parentCommenterHex)
+	if err != nil {
+		logger.Errorf("cannot scan commenterHex and parentCommenterHex: %v", err)
+		return
+	}
+
+	// No reply notification emails for anonymous users.
+	if parentCommenterHex == "anonymous" {
+		return
+	}
+
+	// No reply notification email for self replies.
+	if parentCommenterHex == commenterHex {
+		return
+	}
+
+	pc, err := commenterGetByHex(parentCommenterHex)
+	if err != nil {
+		logger.Errorf("cannot get commenter to send email notification: %v", err)
+		return
+	}
+
+	var commenterName string
+	if commenterHex == "anonymous" {
+		commenterName = "Anonymous"
+	} else {
+		c, err := commenterGetByHex(commenterHex)
+		if err != nil {
+			logger.Errorf("cannot get commenter to send email notification: %v", err)
+			return
+		}
+		commenterName = c.Name
+	}
+
+	// We'll check if they want to receive reply notifications later at the time
+	// of sending.
+	emailNotificationEnqueue(emailNotification{
+		Email:         pc.Email,
+		CommenterName: commenterName,
+		Domain:        d.Domain,
+		Path:          path,
+		Title:         title,
+		CommentHex:    commentHex,
+		Kind:          "reply",
+	})
+}
+
+func emailNotificationNew(d domain, path string, commenterHex string, commentHex string, parentHex string, state string) {
+	p, err := pageGet(d.Domain, path)
+	if err != nil {
+		logger.Errorf("cannot get page to send email notification: %v", err)
+		return
+	}
+
+	if p.Title == "" {
+		p.Title, err = pageTitleUpdate(d.Domain, path)
+		if err != nil {
+			logger.Errorf("cannot update/get page title to send email notification: %v", err)
+			return
+		}
+	}
+
+	emailNotificationModerator(d, path, p.Title, commenterHex, commentHex, state)
+	emailNotificationReply(d, path, p.Title, commenterHex, commentHex, parentHex, state)
+}

api/email_notification_send.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"html/template"
+)
+
+func emailNotificationSend(em string, kind string, notifications []emailNotification) {
+	if len(notifications) == 0 {
+		return
+	}
+
+	e, err := emailGet(em)
+	if err != nil {
+		logger.Errorf("cannot get email: %v", err)
+		return
+	}
+
+	messages := []emailNotificationText{}
+	for _, notification := range notifications {
+		statement := `
+			SELECT html
+			FROM comments
+			WHERE commentHex = $1;
+		`
+		row := db.QueryRow(statement, notification.CommentHex)
+
+		var html string
+		if err = row.Scan(&html); err != nil {
+			// the comment was deleted?
+			// TODO: is this the only error?
+			return
+		}
+
+		messages = append(messages, emailNotificationText{
+			emailNotification: notification,
+			Html:              template.HTML(html),
+		})
+	}
+
+	statement := `
+		SELECT name
+		FROM commenters
+		WHERE email = $1;
+	`
+	row := db.QueryRow(statement, em)
+
+	var name string
+	if err := row.Scan(&name); err != nil {
+		// The moderator has probably not created a commenter account. Let's just
+		// use their email as name.
+		name = nameFromEmail(em)
+	}
+
+	if err := emailNotificationPendingReset(em); err != nil {
+		logger.Errorf("cannot reset after email notification: %v", err)
+		return
+	}
+
+	if err := smtpEmailNotification(em, name, e.UnsubscribeSecretHex, messages, kind); err != nil {
+		logger.Errorf("cannot send email notification: %v", err)
+		return
+	}
+}

api/email_update.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"net/http"
+)
+
+func emailUpdate(e email) error {
+	statement := `
+		UPDATE emails
+		SET sendReplyNotifications = $3, sendModeratorNotifications = $4
+		WHERE email = $1 AND unsubscribeSecretHex = $2;
+	`
+	_, err := db.Exec(statement, e.Email, e.UnsubscribeSecretHex, e.SendReplyNotifications, e.SendModeratorNotifications)
+	if err != nil {
+		logger.Errorf("error updating email: %v", err)
+		return errorInternal
+	}
+
+	return nil
+}
+
+func emailUpdateHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		Email *email `json:"email"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if err := emailUpdate(*x.Email); err != nil {
+		bodyMarshal(w, response{"success": true, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true})
+}

api/errors.go

@@ -42,3 +42,7 @@ var errorInvalidConfigFile = errors.New("Invalid config file.")
 var errorInvalidConfigValue = errors.New("Invalid config value.")
 var errorNewOwnerForbidden = errors.New("New user registrations are forbidden and closed.")
 var errorThreadLocked = errors.New("This thread is locked. You cannot add new comments.")
+var errorDatabaseMigration = errors.New("Encountered error applying database migration.")
+var errorNoSuchUnsubscribeSecretHex = errors.New("Invalid unsubscribe link.")
+var errorEmptyPaths = errors.New("Empty paths field.")
+var errorInvalidDomain = errors.New("Invalid domain name. Do not include the URL path after the forward slash.")

api/main.go

@@ -2,6 +2,7 @@ package main
 
 func main() {
 	exitIfError(loggerCreate())
+	exitIfError(versionPrint())
 	exitIfError(configParse())
 	exitIfError(dbConnect(5))
 	exitIfError(migrate())
@@ -9,8 +10,13 @@ func main() {
 	exitIfError(smtpTemplatesLoad())
 	exitIfError(oauthConfigure())
 	exitIfError(markdownRendererCreate())
+	exitIfError(emailNotificationPendingResetAll())
+	exitIfError(emailNotificationBegin())
 	exitIfError(sigintCleanupSetup())
 	exitIfError(versionCheckStart())
+	exitIfError(domainExportCleanupBegin())
+	exitIfError(viewsCleanupBegin())
+	exitIfError(ssoTokenCleanupBegin())
 
 	exitIfError(routesServe())
 }

api/oauth.go

@@ -2,14 +2,27 @@ package main
 
 import ()
 
-var configuredOauths []string
+var googleConfigured bool
+var twitterConfigured bool
+var githubConfigured bool
+var gitlabConfigured bool
 
 func oauthConfigure() error {
-	configuredOauths = []string{}
-
 	if err := googleOauthConfigure(); err != nil {
 		return err
 	}
 
+	if err := twitterOauthConfigure(); err != nil {
+		return err
+	}
+
+	if err := githubOauthConfigure(); err != nil {
+		return err
+	}
+
+	if err := gitlabOauthConfigure(); err != nil {
+		return err
+	}
+
 	return nil
 }

api/oauth_github.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/github"
+	"os"
+)
+
+var githubConfig *oauth2.Config
+
+func githubOauthConfigure() error {
+	githubConfig = nil
+	if os.Getenv("GITHUB_KEY") == "" && os.Getenv("GITHUB_SECRET") == "" {
+		return nil
+	}
+
+	if os.Getenv("GITHUB_KEY") == "" {
+		logger.Errorf("COMMENTO_GITHUB_KEY not configured, but COMMENTO_GITHUB_SECRET is set")
+		return errorOauthMisconfigured
+	}
+
+	if os.Getenv("GITHUB_SECRET") == "" {
+		logger.Errorf("COMMENTO_GITHUB_SECRET not configured, but COMMENTO_GITHUB_KEY is set")
+		return errorOauthMisconfigured
+	}
+
+	logger.Infof("loading github OAuth config")
+
+	githubConfig = &oauth2.Config{
+		RedirectURL:  os.Getenv("ORIGIN") + "/api/oauth/github/callback",
+		ClientID:     os.Getenv("GITHUB_KEY"),
+		ClientSecret: os.Getenv("GITHUB_SECRET"),
+		Scopes: []string{
+			"read:user",
+			"user:email",
+		},
+		Endpoint: github.Endpoint,
+	}
+
+	githubConfigured = true
+
+	return nil
+}

api/oauth_github_callback.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"golang.org/x/oauth2"
+	"io/ioutil"
+	"net/http"
+)
+
+func githubGetPrimaryEmail(accessToken string) (string, error) {
+	resp, err := http.Get("https://api.github.com/user/emails?access_token=" + accessToken)
+	defer resp.Body.Close()
+
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errorCannotReadResponse
+	}
+
+	user := []map[string]interface{}{}
+	if err := json.Unmarshal(contents, &user); err != nil {
+		logger.Errorf("error unmarshaling github user: %v", err)
+		return "", errorInternal
+	}
+
+	nonPrimaryEmail := ""
+	for _, email := range user {
+		nonPrimaryEmail = email["email"].(string)
+		if email["primary"].(bool) {
+			return email["email"].(string), nil
+		}
+	}
+
+	return nonPrimaryEmail, nil
+}
+
+func githubCallbackHandler(w http.ResponseWriter, r *http.Request) {
+	commenterToken := r.FormValue("state")
+	code := r.FormValue("code")
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	token, err := githubConfig.Exchange(oauth2.NoContext, code)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	email, err := githubGetPrimaryEmail(token.AccessToken)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	resp, err := http.Get("https://api.github.com/user?access_token=" + token.AccessToken)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+	defer resp.Body.Close()
+
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", errorCannotReadResponse.Error())
+		return
+	}
+
+	user := make(map[string]interface{})
+	if err := json.Unmarshal(contents, &user); err != nil {
+		fmt.Fprintf(w, "Error: %s", errorInternal.Error())
+		return
+	}
+
+	if email == "" {
+		if user["email"] == nil {
+			fmt.Fprintf(w, "Error: no email address returned by Github")
+			return
+		}
+
+		email = user["email"].(string)
+	}
+
+	if user["name"] == nil {
+		fmt.Fprintf(w, "Error: no name returned by Github")
+		return
+	}
+
+	name := user["name"].(string)
+
+	link := "undefined"
+	if user["html_url"] != nil {
+		link = user["html_url"].(string)
+	}
+
+	photo := "undefined"
+	if user["avatar_url"] != nil {
+		photo = user["avatar_url"].(string)
+	}
+
+	c, err := commenterGetByEmail("github", email)
+	if err != nil && err != errorNoSuchCommenter {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	var commenterHex string
+
+	// TODO: in case of returning users, update the information we have on record?
+	if err == errorNoSuchCommenter {
+		commenterHex, err = commenterNew(email, name, link, photo, "github", "")
+		if err != nil {
+			fmt.Fprintf(w, "Error: %s", err.Error())
+			return
+		}
+	} else {
+		commenterHex = c.CommenterHex
+	}
+
+	if err := commenterSessionUpdate(commenterToken, commenterHex); err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	fmt.Fprintf(w, "<html><script>window.parent.close()</script></html>")
+}

api/oauth_github_redirect.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func githubRedirectHandler(w http.ResponseWriter, r *http.Request) {
+	if githubConfig == nil {
+		logger.Errorf("github oauth access attempt without configuration")
+		fmt.Fprintf(w, "error: this website has not configured github OAuth")
+		return
+	}
+
+	commenterToken := r.FormValue("commenterToken")
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "error: %s\n", err.Error())
+		return
+	}
+
+	url := githubConfig.AuthCodeURL(commenterToken)
+	http.Redirect(w, r, url, http.StatusFound)
+}

api/oauth_gitlab.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/gitlab"
+	"os"
+)
+
+var gitlabConfig *oauth2.Config
+
+func gitlabOauthConfigure() error {
+	gitlabConfig = nil
+	if os.Getenv("GITLAB_KEY") == "" && os.Getenv("GITLAB_SECRET") == "" {
+		return nil
+	}
+
+	if os.Getenv("GITLAB_KEY") == "" {
+		logger.Errorf("COMMENTO_GITLAB_KEY not configured, but COMMENTO_GITLAB_SECRET is set")
+		return errorOauthMisconfigured
+	}
+
+	if os.Getenv("GITLAB_SECRET") == "" {
+		logger.Errorf("COMMENTO_GITLAB_SECRET not configured, but COMMENTO_GITLAB_KEY is set")
+		return errorOauthMisconfigured
+	}
+
+	logger.Infof("loading gitlab OAuth config")
+
+	gitlabConfig = &oauth2.Config{
+		RedirectURL:  os.Getenv("ORIGIN") + "/api/oauth/gitlab/callback",
+		ClientID:     os.Getenv("GITLAB_KEY"),
+		ClientSecret: os.Getenv("GITLAB_SECRET"),
+		Scopes: []string{
+			"read_user",
+		},
+		Endpoint: gitlab.Endpoint,
+	}
+
+	gitlabConfigured = true
+
+	return nil
+}

api/oauth_gitlab_callback.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"golang.org/x/oauth2"
+	"io/ioutil"
+	"net/http"
+)
+
+func gitlabCallbackHandler(w http.ResponseWriter, r *http.Request) {
+	commenterToken := r.FormValue("state")
+	code := r.FormValue("code")
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	token, err := gitlabConfig.Exchange(oauth2.NoContext, code)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	resp, err := http.Get("https://gitlab.com/api/v4/user?access_token=" + token.AccessToken)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+	logger.Infof("%v", resp.StatusCode)
+	defer resp.Body.Close()
+
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s", errorCannotReadResponse.Error())
+		return
+	}
+
+	user := make(map[string]interface{})
+	if err := json.Unmarshal(contents, &user); err != nil {
+		fmt.Fprintf(w, "Error: %s", errorInternal.Error())
+		return
+	}
+
+	if user["email"] == nil {
+		fmt.Fprintf(w, "Error: no email address returned by Gitlab")
+		return
+	}
+
+	email := user["email"].(string)
+
+	if user["name"] == nil {
+		fmt.Fprintf(w, "Error: no name returned by Gitlab")
+		return
+	}
+
+	name := user["name"].(string)
+
+	link := "undefined"
+	if user["web_url"] != nil {
+		link = user["web_url"].(string)
+	}
+
+	photo := "undefined"
+	if user["avatar_url"] != nil {
+		photo = user["avatar_url"].(string)
+	}
+
+	c, err := commenterGetByEmail("gitlab", email)
+	if err != nil && err != errorNoSuchCommenter {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	var commenterHex string
+
+	// TODO: in case of returning users, update the information we have on record?
+	if err == errorNoSuchCommenter {
+		commenterHex, err = commenterNew(email, name, link, photo, "gitlab", "")
+		if err != nil {
+			fmt.Fprintf(w, "Error: %s", err.Error())
+			return
+		}
+	} else {
+		commenterHex = c.CommenterHex
+	}
+
+	if err := commenterSessionUpdate(commenterToken, commenterHex); err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	fmt.Fprintf(w, "<html><script>window.parent.close()</script></html>")
+}

api/oauth_gitlab_redirect.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func gitlabRedirectHandler(w http.ResponseWriter, r *http.Request) {
+	if gitlabConfig == nil {
+		logger.Errorf("gitlab oauth access attempt without configuration")
+		fmt.Fprintf(w, "error: this website has not configured gitlab OAuth")
+		return
+	}
+
+	commenterToken := r.FormValue("commenterToken")
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "error: %s\n", err.Error())
+		return
+	}
+
+	url := gitlabConfig.AuthCodeURL(commenterToken)
+	http.Redirect(w, r, url, http.StatusFound)
+}

api/oauth_google.go

@@ -37,7 +37,7 @@ func googleOauthConfigure() error {
 		Endpoint: google.Endpoint,
 	}
 
-	configuredOauths = append(configuredOauths, "google")
+	googleConfigured = true
 
 	return nil
 }

api/oauth_google_callback.go

@@ -39,7 +39,14 @@ func googleCallbackHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	c, err := commenterGetByEmail("google", user["email"].(string))
+	if user["email"] == nil {
+		fmt.Fprintf(w, "Error: no email address returned by Github")
+		return
+	}
+
+	email := user["email"].(string)
+
+	c, err := commenterGetByEmail("google", email)
 	if err != nil && err != errorNoSuchCommenter {
 		fmt.Fprintf(w, "Error: %s", err.Error())
 		return
@@ -49,14 +56,6 @@ func googleCallbackHandler(w http.ResponseWriter, r *http.Request) {
 
 	// TODO: in case of returning users, update the information we have on record?
 	if err == errorNoSuchCommenter {
-		var email string
-		if _, ok := user["email"]; ok {
-			email = user["email"].(string)
-		} else {
-			fmt.Fprintf(w, "Error: %s", errorInvalidEmail.Error())
-			return
-		}
-
 		var link string
 		if val, ok := user["link"]; ok {
 			link = val.(string)

api/oauth_sso.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"time"
+)
+
+type ssoPayload struct {
+	Domain string `json:"domain"`
+	Token  string `json:"token"`
+	Email  string `json:"email"`
+	Name   string `json:"name"`
+	Link   string `json:"link"`
+	Photo  string `json:"photo"`
+}
+
+func ssoTokenNew(domain string, commenterToken string) (string, error) {
+	token, err := randomHex(32)
+	if err != nil {
+		logger.Errorf("error generating SSO token hex: %v", err)
+		return "", errorInternal
+	}
+
+	statement := `
+		INSERT INTO
+		ssoTokens (token, domain, commenterToken, creationDate)
+		VALUES    ($1,    $2,     $3,             $4          );
+	`
+	_, err = db.Exec(statement, token, domain, commenterToken, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("error inserting SSO token: %v", err)
+		return "", errorInternal
+	}
+
+	return token, nil
+}
+
+func ssoTokenExtract(token string) (string, string, error) {
+	statement := `
+		SELECT domain, commenterToken
+		FROM ssoTokens
+		WHERE token = $1;
+	`
+	row := db.QueryRow(statement, token)
+
+	var domain string
+	var commenterToken string
+	if err := row.Scan(&domain, &commenterToken); err != nil {
+		return "", "", errorNoSuchToken
+	}
+
+	statement = `
+		DELETE FROM ssoTokens
+		WHERE token = $1;
+	`
+	if _, err := db.Exec(statement, token); err != nil {
+		logger.Errorf("cannot delete SSO token after usage: %v", err)
+		return "", "", errorInternal
+	}
+
+	return domain, commenterToken, nil
+}

api/oauth_sso_callback.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+func ssoCallbackHandler(w http.ResponseWriter, r *http.Request) {
+	payloadHex := r.FormValue("payload")
+	signature := r.FormValue("hmac")
+
+	payloadBytes, err := hex.DecodeString(payloadHex)
+	if err != nil {
+		fmt.Fprintf(w, "Error: invalid JSON payload hex encoding: %s\n", err.Error())
+		return
+	}
+
+	signatureBytes, err := hex.DecodeString(signature)
+	if err != nil {
+		fmt.Fprintf(w, "Error: invalid HMAC signature hex encoding: %s\n", err.Error())
+		return
+	}
+
+	payload := ssoPayload{}
+	err = json.Unmarshal(payloadBytes, &payload)
+	if err != nil {
+		fmt.Fprintf(w, "Error: cannot unmarshal JSON payload: %s\n", err.Error())
+		return
+	}
+
+	if payload.Token == "" || payload.Email == "" || payload.Name == "" {
+		fmt.Fprintf(w, "Error: %s\n", errorMissingField.Error())
+		return
+	}
+
+	if payload.Link == "" {
+		payload.Link = "undefined"
+	}
+
+	if payload.Photo == "" {
+		payload.Photo = "undefined"
+	}
+
+	domain, commenterToken, err := ssoTokenExtract(payload.Token)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	d, err := domainGet(domain)
+	if err != nil {
+		if err == errorNoSuchDomain {
+			fmt.Fprintf(w, "Error: %s\n", err.Error())
+		} else {
+			logger.Errorf("cannot get domain for SSO: %v", err)
+			fmt.Fprintf(w, "Error: %s\n", errorInternal.Error())
+		}
+		return
+	}
+
+	if d.SsoSecret == "" || d.SsoUrl == "" {
+		fmt.Fprintf(w, "Error: %s\n", errorMissingConfig.Error())
+		return
+	}
+
+	key, err := hex.DecodeString(d.SsoSecret)
+	if err != nil {
+		logger.Errorf("cannot decode SSO secret as hex: %v", err)
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	h := hmac.New(sha256.New, key)
+	h.Write(payloadBytes)
+	expectedSignatureBytes := h.Sum(nil)
+	if !hmac.Equal(expectedSignatureBytes, signatureBytes) {
+		fmt.Fprintf(w, "Error: HMAC signature verification failed\n")
+		return
+	}
+
+	_, err = commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	c, err := commenterGetByEmail("sso:"+domain, payload.Email)
+	if err != nil && err != errorNoSuchCommenter {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	var commenterHex string
+
+	// TODO: in case of returning users, update the information we have on record?
+	if err == errorNoSuchCommenter {
+		commenterHex, err = commenterNew(payload.Email, payload.Name, payload.Link, payload.Photo, "sso:"+domain, "")
+		if err != nil {
+			fmt.Fprintf(w, "Error: %s", err.Error())
+			return
+		}
+	} else {
+		commenterHex = c.CommenterHex
+	}
+
+	if err = commenterSessionUpdate(commenterToken, commenterHex); err != nil {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	fmt.Fprintf(w, "<html><script>window.parent.close()</script></html>")
+}

api/oauth_sso_redirect.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"net/url"
+)
+
+func ssoRedirectHandler(w http.ResponseWriter, r *http.Request) {
+	commenterToken := r.FormValue("commenterToken")
+	domain := r.Header.Get("Referer")
+
+	if commenterToken == "" {
+		fmt.Fprintf(w, "Error: %s\n", errorMissingField.Error())
+		return
+	}
+
+	domain = domainStrip(domain)
+	if domain == "" {
+		fmt.Fprintf(w, "Error: No Referer header found in request\n")
+		return
+	}
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	d, err := domainGet(domain)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s\n", errorNoSuchDomain.Error())
+		return
+	}
+
+	if !d.SsoProvider {
+		fmt.Fprintf(w, "Error: SSO not configured for %s\n", domain)
+		return
+	}
+
+	if d.SsoSecret == "" || d.SsoUrl == "" {
+		fmt.Fprintf(w, "Error: %s\n", errorMissingConfig.Error())
+		return
+	}
+
+	key, err := hex.DecodeString(d.SsoSecret)
+	if err != nil {
+		logger.Errorf("cannot decode SSO secret as hex: %v", err)
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	token, err := ssoTokenNew(domain, commenterToken)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	tokenBytes, err := hex.DecodeString(token)
+	if err != nil {
+		logger.Errorf("cannot decode hex token: %v", err)
+		fmt.Fprintf(w, "Error: %s\n", errorInternal.Error())
+		return
+	}
+
+	h := hmac.New(sha256.New, key)
+	h.Write(tokenBytes)
+	signature := hex.EncodeToString(h.Sum(nil))
+
+	u, err := url.Parse(d.SsoUrl)
+	if err != nil {
+		// this should really not be happening; we're checking if the
+		// passed URL is valid at domain update
+		logger.Errorf("cannot parse URL: %v", err)
+		fmt.Fprintf(w, "Error: %s\n", errorInternal.Error())
+		return
+	}
+
+	q := u.Query()
+	q.Set("token", token)
+	q.Set("hmac", signature)
+	u.RawQuery = q.Encode()
+
+	http.Redirect(w, r, u.String(), http.StatusFound)
+}

api/oauth_twitter.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"github.com/gomodule/oauth1/oauth"
+	"os"
+	"sync"
+)
+
+type twitterOauthState struct {
+	CommenterToken string
+	Cred           *oauth.Credentials
+}
+
+var twitterClient *oauth.Client
+var twitterCredMapLock sync.RWMutex
+var twitterCredMap map[string]twitterOauthState
+
+func twitterOauthConfigure() error {
+	twitterClient = nil
+	if os.Getenv("TWITTER_KEY") == "" && os.Getenv("TWITTER_SECRET") == "" {
+		return nil
+	}
+
+	if os.Getenv("TWITTER_KEY") == "" {
+		logger.Errorf("COMMENTO_TWITTER_KEY not configured, but COMMENTO_TWITTER_SECRET is set")
+		return errorOauthMisconfigured
+	}
+
+	if os.Getenv("TWITTER_SECRET") == "" {
+		logger.Errorf("COMMENTO_TWITTER_SECRET not configured, but COMMENTO_TWITTER_KEY is set")
+		return errorOauthMisconfigured
+	}
+
+	logger.Infof("loading twitter OAuth config")
+
+	twitterClient = &oauth.Client{
+		TemporaryCredentialRequestURI: "https://api.twitter.com/oauth/request_token",
+		ResourceOwnerAuthorizationURI: "https://api.twitter.com/oauth/authenticate",
+		TokenRequestURI:               "https://api.twitter.com/oauth/access_token",
+		Credentials: oauth.Credentials{
+			Token:  os.Getenv("TWITTER_KEY"),
+			Secret: os.Getenv("TWITTER_SECRET"),
+		},
+	}
+
+	twitterCredMap = make(map[string]twitterOauthState, 1e3)
+
+	twitterConfigured = true
+
+	return nil
+}

api/oauth_twitter_callback.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+)
+
+func twitterCallbackHandler(w http.ResponseWriter, r *http.Request) {
+	token := r.FormValue("oauth_token")
+	verifier := r.FormValue("oauth_verifier")
+
+	twitterCredMapLock.RLock()
+	s, ok := twitterCredMap[token]
+	twitterCredMapLock.RUnlock()
+
+	commenterToken := s.CommenterToken
+
+	if !ok {
+		fmt.Fprintf(w, "no such token/verifier combination found")
+		return
+	}
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	x, _, err := twitterClient.RequestToken(nil, s.Cred, verifier)
+	if err != nil {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	twitterCredMapLock.Lock()
+	delete(twitterCredMap, token)
+	twitterCredMapLock.Unlock()
+
+	resp, err := twitterClient.Get(nil, x, "https://api.twitter.com/1.1/account/verify_credentials.json", url.Values{"include_email": {"true"}})
+	if err != nil {
+		fmt.Fprintf(w, "Error getting email: %s\n", err.Error())
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		msg, _ := ioutil.ReadAll(resp.Body)
+		fmt.Fprintf(w, "Error: status %d: %s\n", resp.StatusCode, msg)
+		return
+	}
+
+	var res map[string]interface{}
+	if err = json.NewDecoder(resp.Body).Decode(&res); err != nil {
+		fmt.Fprintf(w, "Error: %s\n", err.Error())
+		return
+	}
+
+	if res["email"] == nil {
+		fmt.Fprintf(w, "Error: no email address returned by Twitter")
+		return
+	}
+
+	email := res["email"].(string)
+
+	if res["name"] == nil {
+		fmt.Fprintf(w, "Error: no name returned by Twitter")
+		return
+	}
+
+	name := res["name"].(string)
+
+	link := "undefined"
+	photo := "undefined"
+	if res["handle"] != nil {
+		handle := res["screen_name"].(string)
+		link = "https://twitter.com/" + handle
+		photo = "https://twitter.com/" + handle + "/profile_image"
+	}
+
+	c, err := commenterGetByEmail("twitter", email)
+	if err != nil && err != errorNoSuchCommenter {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	var commenterHex string
+
+	// TODO: in case of returning users, update the information we have on record?
+	if err == errorNoSuchCommenter {
+		commenterHex, err = commenterNew(email, name, link, photo, "twitter", "")
+		if err != nil {
+			fmt.Fprintf(w, "Error: %s", err.Error())
+			return
+		}
+	} else {
+		commenterHex = c.CommenterHex
+	}
+
+	if err := commenterSessionUpdate(commenterToken, commenterHex); err != nil {
+		fmt.Fprintf(w, "Error: %s", err.Error())
+		return
+	}
+
+	fmt.Fprintf(w, "<html><script>window.parent.close()</script></html>")
+}

api/oauth_twitter_redirect.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+)
+
+func twitterRedirectHandler(w http.ResponseWriter, r *http.Request) {
+	if twitterClient == nil {
+		logger.Errorf("twitter oauth access attempt without configuration")
+		fmt.Fprintf(w, "error: this website has not configured twitter OAuth")
+		return
+	}
+
+	commenterToken := r.FormValue("commenterToken")
+
+	_, err := commenterGetByCommenterToken(commenterToken)
+	if err != nil && err != errorNoSuchToken {
+		fmt.Fprintf(w, "error: %s\n", err.Error())
+		return
+	}
+
+	cred, err := twitterClient.RequestTemporaryCredentials(nil, os.Getenv("ORIGIN")+"/api/oauth/twitter/callback", nil)
+	if err != nil {
+		logger.Errorf("cannot get temporary twitter credentials: %v", err)
+		fmt.Fprintf(w, "error: %v", errorInternal.Error())
+		return
+	}
+
+	twitterCredMapLock.Lock()
+	twitterCredMap[cred.Token] = twitterOauthState{
+		CommenterToken: commenterToken,
+		Cred:           cred,
+	}
+	twitterCredMapLock.Unlock()
+
+	http.Redirect(w, r, twitterClient.AuthorizationURL(cred, nil), http.StatusFound)
+}

api/owner_get.go

@@ -31,8 +31,8 @@ func ownerGetByOwnerToken(ownerToken string) (owner, error) {
 	statement := `
     SELECT ownerHex, email, name, confirmedEmail, joinDate
 		FROM owners
-		WHERE email IN (
-			SELECT email FROM ownerSessions
+		WHERE ownerHex IN (
+			SELECT ownerHex FROM ownerSessions
 			WHERE ownerToken = $1
 		);
 	`
@@ -46,3 +46,24 @@ func ownerGetByOwnerToken(ownerToken string) (owner, error) {
 
 	return o, nil
 }
+
+func ownerGetByOwnerHex(ownerHex string) (owner, error) {
+	if ownerHex == "" {
+		return owner{}, errorMissingField
+	}
+
+	statement := `
+    SELECT ownerHex, email, name, confirmedEmail, joinDate
+		FROM owners
+		WHERE ownerHex = $1;
+	`
+	row := db.QueryRow(statement, ownerHex)
+
+	var o owner
+	if err := row.Scan(&o.OwnerHex, &o.Email, &o.Name, &o.ConfirmedEmail, &o.JoinDate); err != nil {
+		logger.Errorf("cannot scan owner: %v\n", err)
+		return owner{}, errorInternal
+	}
+
+	return o, nil
+}

api/owner_new.go

@@ -16,6 +16,14 @@ func ownerNew(email string, name string, password string) (string, error) {
 		return "", errorNewOwnerForbidden
 	}
 
+	if _, err := ownerGetByEmail(email); err == nil {
+		return "", errorEmailAlreadyExists
+	}
+
+	if err := emailNew(email); err != nil {
+		return "", errorInternal
+	}
+
 	ownerHex, err := randomHex(32)
 	if err != nil {
 		logger.Errorf("cannot generate ownerHex: %v", err)

api/owner_reset_password.go

@@ -18,8 +18,9 @@ func ownerResetPassword(resetHex string, password string) error {
 
 	statement := `
 		UPDATE owners SET passwordHash=$1
-		WHERE email IN (
-			SELECT email FROM ownerResetHexes
+		WHERE ownerHex = (
+			SELECT ownerHex
+			FROM ownerResetHexes
 			WHERE resetHex=$2
 		);
 	`

api/page.go

@@ -3,8 +3,10 @@ package main
 import ()
 
 type page struct {
-	Domain       string `json:"domain"`
-	Path         string `json:"path"`
-	IsLocked     bool   `json:"isLocked"`
-	CommentCount int    `json:"commentCount"`
+	Domain           string `json:"domain"`
+	Path             string `json:"path"`
+	IsLocked         bool   `json:"isLocked"`
+	CommentCount     int    `json:"commentCount"`
+	StickyCommentHex string `json:"stickyCommentHex"`
+	Title            string `json:"title"`
 }

api/page_get.go

@@ -11,20 +11,22 @@ func pageGet(domain string, path string) (page, error) {
 	}
 
 	statement := `
-		SELECT isLocked, commentCount
+		SELECT isLocked, commentCount, stickyCommentHex, title
 		FROM pages
 		WHERE domain=$1 AND path=$2;
 	`
 	row := db.QueryRow(statement, domain, path)
 
 	p := page{Domain: domain, Path: path}
-	if err := row.Scan(&p.IsLocked, &p.CommentCount); err != nil {
+	if err := row.Scan(&p.IsLocked, &p.CommentCount, &p.StickyCommentHex, &p.Title); err != nil {
 		if err == sql.ErrNoRows {
 			// If there haven't been any comments, there won't be a record for this
 			// page. The sane thing to do is return defaults.
 			// TODO: the defaults are hard-coded in two places: here and the schema
 			p.IsLocked = false
 			p.CommentCount = 0
+			p.StickyCommentHex = "none"
+			p.Title = ""
 		} else {
 			logger.Errorf("error scanning page: %v", err)
 			return page{}, errorInternal

api/page_title.go

@@ -0,0 +1,28 @@
+package main
+
+import ()
+
+func pageTitleUpdate(domain string, path string) (string, error) {
+	title, err := htmlTitleGet("http://" + domain + path)
+	if err != nil {
+		// This could fail due to a variety of reasons that we can't control such
+		// as the user's URL 404 or something, so let's not pollute the error log
+		// with messages. Just use a sane title. Maybe we'll have the ability to
+		// retry later.
+		logger.Errorf("%v", err)
+		title = domain
+	}
+
+	statement := `
+		UPDATE pages
+		SET title = $3
+		WHERE domain = $1 AND path = $2;
+	`
+	_, err = db.Exec(statement, domain, path, title)
+	if err != nil {
+		logger.Errorf("cannot update pages table with title: %v", err)
+		return "", err
+	}
+
+	return title, nil
+}

api/page_update.go

@@ -13,12 +13,12 @@ func pageUpdate(p page) error {
 	//   commentCount
 	statement := `
 		INSERT INTO
-		pages  (domain, path, isLocked)
-		VALUES ($1,     $2,   $3    )
+		pages  (domain, path, isLocked, stickyCommentHex)
+		VALUES ($1,     $2,   $3,       $4              )
 		ON CONFLICT (domain, path) DO
-			UPDATE SET isLocked = $3;
+			UPDATE SET isLocked = $3, stickyCommentHex = $4;
 	`
-	_, err := db.Exec(statement, p.Domain, p.Path, p.IsLocked)
+	_, err := db.Exec(statement, p.Domain, p.Path, p.IsLocked, p.StickyCommentHex)
 	if err != nil {
 		logger.Errorf("error setting page attributes: %v", err)
 		return errorInternal

api/router_api.go

@@ -14,21 +14,42 @@ func apiRouterInit(router *mux.Router) error {
 
 	router.HandleFunc("/api/domain/new", domainNewHandler).Methods("POST")
 	router.HandleFunc("/api/domain/delete", domainDeleteHandler).Methods("POST")
+	router.HandleFunc("/api/domain/clear", domainClearHandler).Methods("POST")
+	router.HandleFunc("/api/domain/sso/new", domainSsoSecretNewHandler).Methods("POST")
 	router.HandleFunc("/api/domain/list", domainListHandler).Methods("POST")
 	router.HandleFunc("/api/domain/update", domainUpdateHandler).Methods("POST")
 	router.HandleFunc("/api/domain/moderator/new", domainModeratorNewHandler).Methods("POST")
 	router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST")
 	router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST")
 	router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET")
 
 	router.HandleFunc("/api/commenter/token/new", commenterTokenNewHandler).Methods("GET")
 	router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST")
 	router.HandleFunc("/api/commenter/login", commenterLoginHandler).Methods("POST")
 	router.HandleFunc("/api/commenter/self", commenterSelfHandler).Methods("POST")
+	router.HandleFunc("/api/commenter/photo", commenterPhotoHandler).Methods("GET")
+
+	router.HandleFunc("/api/email/get", emailGetHandler).Methods("POST")
+	router.HandleFunc("/api/email/update", emailUpdateHandler).Methods("POST")
+	router.HandleFunc("/api/email/moderate", emailModerateHandler).Methods("GET")
 
 	router.HandleFunc("/api/oauth/google/redirect", googleRedirectHandler).Methods("GET")
 	router.HandleFunc("/api/oauth/google/callback", googleCallbackHandler).Methods("GET")
 
+	router.HandleFunc("/api/oauth/github/redirect", githubRedirectHandler).Methods("GET")
+	router.HandleFunc("/api/oauth/github/callback", githubCallbackHandler).Methods("GET")
+
+	router.HandleFunc("/api/oauth/twitter/redirect", twitterRedirectHandler).Methods("GET")
+	router.HandleFunc("/api/oauth/twitter/callback", twitterCallbackHandler).Methods("GET")
+
+	router.HandleFunc("/api/oauth/gitlab/redirect", gitlabRedirectHandler).Methods("GET")
+	router.HandleFunc("/api/oauth/gitlab/callback", gitlabCallbackHandler).Methods("GET")
+
+	router.HandleFunc("/api/oauth/sso/redirect", ssoRedirectHandler).Methods("GET")
+	router.HandleFunc("/api/oauth/sso/callback", ssoCallbackHandler).Methods("GET")
+
 	router.HandleFunc("/api/comment/new", commentNewHandler).Methods("POST")
 	router.HandleFunc("/api/comment/list", commentListHandler).Methods("POST")
 	router.HandleFunc("/api/comment/count", commentCountHandler).Methods("POST")

api/router_static.go

@@ -1,41 +1,82 @@
 package main
 
 import (
-	"bytes"
-	"fmt"
 	"github.com/gorilla/mux"
-	"html/template"
 	"io/ioutil"
 	"mime"
 	"net/http"
 	"os"
 	"path"
+	"strings"
 )
 
 func redirectLogin(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, os.Getenv("ORIGIN")+"/login", 301)
 }
 
-type staticAssetPlugs struct {
-	Origin string
-}
-
-type staticHtmlPlugs struct {
+type staticPlugs struct {
 	Origin    string
 	CdnPrefix string
-	Footer    template.HTML
+	Footer    string
+}
+
+var asset map[string][]byte = make(map[string][]byte)
+var contentType map[string]string = make(map[string]string)
+var footer string
+var compress bool
+
+func fileDetemplate(f string) ([]byte, error) {
+	contents, err := ioutil.ReadFile(f)
+	if err != nil {
+		logger.Errorf("cannot read file %s: %v", f, err)
+		return []byte{}, err
+	}
+
+	x := string(contents)
+	x = strings.Replace(x, "[[[.Origin]]]", os.Getenv("ORIGIN"), -1)
+	x = strings.Replace(x, "[[[.CdnPrefix]]]", os.Getenv("CDN_PREFIX"), -1)
+	x = strings.Replace(x, "[[[.Footer]]]", footer, -1)
+	x = strings.Replace(x, "[[[.Version]]]", version, -1)
+
+	return []byte(x), nil
+}
+
+func footerInit() error {
+	contents, err := fileDetemplate(os.Getenv("STATIC") + "/footer.html")
+	if err != nil {
+		logger.Errorf("cannot init footer: %v", err)
+		return err
+	}
+
+	footer = string(contents)
+	return nil
+}
+
+func fileLoad(f string) ([]byte, error) {
+	b, err := fileDetemplate(f)
+	if err != nil {
+		logger.Errorf("cannot load file %s: %v", f, err)
+		return []byte{}, err
+	}
+
+	if !compress {
+		return b, nil
+	}
+
+	return gzipStatic(b)
 }
 
 func staticRouterInit(router *mux.Router) error {
+	var err error
+
 	subdir := pathStrip(os.Getenv("ORIGIN"))
 
-	asset := make(map[string][]byte)
-	gzippedAsset := make(map[string][]byte)
-
-	for _, dir := range []string{"js", "css", "images"} {
-		sl := string(os.PathSeparator)
-		dir = sl + dir
+	if err = footerInit(); err != nil {
+		logger.Errorf("error initialising static router: %v", err)
+		return err
+	}
 
+	for _, dir := range []string{"/js", "/css", "/images", "/fonts"} {
 		files, err := ioutil.ReadDir(os.Getenv("STATIC") + dir)
 		if err != nil {
 			logger.Errorf("cannot read directory %s%s: %v", os.Getenv("STATIC"), dir, err)
@@ -43,108 +84,48 @@ func staticRouterInit(router *mux.Router) error {
 		}
 
 		for _, file := range files {
-			p := dir + sl + file.Name()
-
-			contents, err := ioutil.ReadFile(os.Getenv("STATIC") + p)
+			f := dir + "/" + file.Name()
+			asset[subdir+f], err = fileLoad(os.Getenv("STATIC") + f)
 			if err != nil {
-				logger.Errorf("cannot read file %s%s: %v", os.Getenv("STATIC"), p, err)
+				logger.Errorf("cannot detemplate %s%s: %v", os.Getenv("STATIC"), f, err)
 				return err
 			}
-
-			prefix := ""
-			if dir == "/js" {
-				prefix = "window.commentoOrigin='" + os.Getenv("ORIGIN") + "';\n"
-				prefix += "window.commentoCdn='" + os.Getenv("CDN_PREFIX") + "';\n"
-			}
-
-			gzip := (os.Getenv("GZIP_STATIC") == "true")
-
-			asset[subdir+p] = []byte(prefix + string(contents))
-			if gzip {
-				gzippedAsset[subdir+p], err = gzipStatic(asset[subdir+p])
-				if err != nil {
-					logger.Errorf("error gzipping %s: %v", p, err)
-					return err
-				}
-			}
-
-			// faster than checking inside the handler
-			if !gzip {
-				router.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) {
-					w.Header().Set("Content-Type", mime.TypeByExtension(path.Ext(r.URL.Path)))
-					w.Write(asset[r.URL.Path])
-				})
-			} else {
-				router.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) {
-					w.Header().Set("Content-Type", mime.TypeByExtension(path.Ext(r.URL.Path)))
-					w.Header().Set("Content-Encoding", "gzip")
-					w.Write(gzippedAsset[r.URL.Path])
-				})
-			}
 		}
 	}
 
-	footer, err := ioutil.ReadFile(os.Getenv("STATIC") + string(os.PathSeparator) + "footer.html")
-	if err != nil {
-		logger.Errorf("cannot read file footer.html: %v", err)
-		return err
-	}
-
 	pages := []string{
-		"login",
-		"forgot",
-		"reset-password",
-		"signup",
-		"confirm-email",
-		"dashboard",
-		"logout",
-	}
-
-	html := make(map[string]string)
-	for _, page := range pages {
-		html[subdir+page] = ""
+		"/login",
+		"/forgot",
+		"/reset-password",
+		"/signup",
+		"/confirm-email",
+		"/unsubscribe",
+		"/dashboard",
+		"/logout",
 	}
 
 	for _, page := range pages {
-		sl := string(os.PathSeparator)
-		page = sl + page
-		file := page + ".html"
-
-		contents, err := ioutil.ReadFile(os.Getenv("STATIC") + file)
+		f := page + ".html"
+		asset[subdir+page], err = fileLoad(os.Getenv("STATIC") + f)
 		if err != nil {
-			logger.Errorf("cannot read file %s%s: %v", os.Getenv("STATIC"), file, err)
+			logger.Errorf("cannot detemplate %s%s: %v", os.Getenv("STATIC"), f, err)
 			return err
 		}
-
-		result := string(contents)
-
-		for {
-			t, err := template.New(page).Delims("[[[", "]]]").Parse(result)
-			if err != nil {
-				logger.Errorf("cannot parse %s%s template: %v", os.Getenv("STATIC"), file, err)
-				return err
-			}
-
-			var buf bytes.Buffer
-			t.Execute(&buf, &staticHtmlPlugs{
-				Origin:    os.Getenv("ORIGIN"),
-				CdnPrefix: os.Getenv("CDN_PREFIX"),
-				Footer:    template.HTML(string(footer)),
-			})
-
-			result = buf.String()
-			if result == html[subdir+page] {
-				break
-			} else {
-				html[subdir+page] = result
-				continue
-			}
-		}
 	}
 
-	for _, page := range pages {
-		router.HandleFunc("/"+page, func(w http.ResponseWriter, r *http.Request) {
-			fmt.Fprint(w, html[r.URL.Path])
+	for p, _ := range asset {
+		if path.Ext(p) != "" {
+			contentType[p] = mime.TypeByExtension(path.Ext(p))
+		} else {
+			contentType[p] = "text/html; charset=utf-8"
+		}
+
+		router.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", contentType[r.URL.Path])
+			if compress {
+				w.Header().Set("Content-Encoding", "gzip")
+			}
+			w.Write(asset[r.URL.Path])
 		})
 	}
 

api/sigint.go

@@ -7,7 +7,14 @@ import (
 )
 
 func sigintCleanup() int {
-	// TODO: close the database connection and do other cleanup jobs
+	if db != nil {
+		err := db.Close()
+		if err == nil {
+			logger.Errorf("cannot close database connection: %v", err)
+			return 1
+		}
+	}
+
 	return 0
 }
 

api/smtp_configure.go

@@ -13,12 +13,16 @@ func smtpConfigure() error {
 	password := os.Getenv("SMTP_PASSWORD")
 	host := os.Getenv("SMTP_HOST")
 	port := os.Getenv("SMTP_PORT")
-	if username == "" || password == "" || host == "" || port == "" {
+	if host == "" || port == "" {
 		logger.Warningf("smtp not configured, no emails will be sent")
 		smtpConfigured = false
 		return nil
 	}
 
+	if username == "" || password == "" {
+		logger.Warningf("no SMTP username/password set, Commento will assume they aren't required")
+	}
+
 	if os.Getenv("SMTP_FROM_ADDRESS") == "" {
 		logger.Errorf("COMMENTO_SMTP_FROM_ADDRESS not set")
 		smtpConfigured = false

api/smtp_domain_export.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportPlugs struct {
+	Origin    string
+	Domain    string
+	ExportHex string
+}
+
+func smtpDomainExport(to string, toName string, domain string, exportHex string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["domain-export"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN"), ExportHex: exportHex})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_domain_export_error.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportErrorPlugs struct {
+	Origin string
+	Domain string
+}
+
+func smtpDomainExportError(to string, toName string, domain string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["data-export-error"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN")})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export error email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_email_notification.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	ht "html/template"
+	"net/smtp"
+	"os"
+	tt "text/template"
+)
+
+type emailNotificationText struct {
+	emailNotification
+	Html ht.HTML
+}
+
+type emailNotificationPlugs struct {
+	Origin               string
+	Kind                 string
+	Subject              string
+	UnsubscribeSecretHex string
+	Notifications        []emailNotificationText
+}
+
+func smtpEmailNotification(to string, toName string, unsubscribeSecretHex string, notifications []emailNotificationText, kind string) error {
+	var subject string
+	if kind == "reply" {
+		var verb string
+		if len(notifications) > 1 {
+			verb = "replies"
+		} else {
+			verb = "reply"
+		}
+		subject = fmt.Sprintf("%d new comment %s", len(notifications), verb)
+	} else {
+		var verb string
+		if len(notifications) > 1 {
+			verb = "comments"
+		} else {
+			verb = "comment"
+		}
+		if kind == "pending-moderation" {
+			subject = fmt.Sprintf("%d new %s pending moderation", len(notifications), verb)
+		} else {
+			subject = fmt.Sprintf("%d new %s on your website", len(notifications), verb)
+		}
+	}
+
+	h, err := tt.New("header").Parse(`MIME-Version: 1.0
+From: Commento <{{.FromAddress}}>
+To: {{.ToName}} <{{.ToAddress}}>
+Content-Type: text/html; charset=UTF-8
+Subject: {{.Subject}}
+
+`)
+
+	var header bytes.Buffer
+	h.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "[Commento] " + subject})
+
+	t, err := ht.ParseFiles(fmt.Sprintf("%s/templates/email-notification.txt", os.Getenv("STATIC")))
+	if err != nil {
+		logger.Errorf("cannot parse %s/templates/email-notification.txt: %v", os.Getenv("STATIC"), err)
+		return errorMalformedTemplate
+	}
+
+	var body bytes.Buffer
+	err = t.Execute(&body, &emailNotificationPlugs{
+		Origin:               os.Getenv("ORIGIN"),
+		Kind:                 kind,
+		Subject:              subject,
+		UnsubscribeSecretHex: unsubscribeSecretHex,
+		Notifications:        notifications,
+	})
+	if err != nil {
+		logger.Errorf("error generating templated HTML for email notification: %v", err)
+		return err
+	}
+
+	err = smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send email notification: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_templates.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"fmt"
-	"html/template"
 	"os"
+	"text/template"
 )
 
 var headerTemplate *template.Template
@@ -20,9 +20,9 @@ var templates map[string]*template.Template
 func smtpTemplatesLoad() error {
 	var err error
 	headerTemplate, err = template.New("header").Parse(`MIME-Version: 1.0
-Content-Type: text/html; charset=UTF-8
-From: {{.FromAddress}}
+From: Commento <{{.FromAddress}}>
 To: {{.ToName}} <{{.ToAddress}}>
+Content-Type: text/plain; charset=UTF-8
 Subject: {{.Subject}}
 
 `)
@@ -31,7 +31,12 @@ Subject: {{.Subject}}
 		return errorMalformedTemplate
 	}
 
-	names := []string{"confirm-hex", "reset-hex"}
+	names := []string{
+		"confirm-hex",
+		"reset-hex",
+		"domain-export",
+		"domain-export-error",
+	}
 
 	templates = make(map[string]*template.Template)
 
@@ -39,9 +44,9 @@ Subject: {{.Subject}}
 	for _, name := range names {
 		var err error
 		templates[name] = template.New(name)
-		templates[name], err = template.ParseFiles(fmt.Sprintf("%s/templates/%s.html", os.Getenv("STATIC"), name))
+		templates[name], err = template.ParseFiles(fmt.Sprintf("%s/templates/%s.txt", os.Getenv("STATIC"), name))
 		if err != nil {
-			logger.Errorf("cannot parse %s/templates/%s.html: %v", os.Getenv("STATIC"), name, err)
+			logger.Errorf("cannot parse %s/templates/%s.txt: %v", os.Getenv("STATIC"), name, err)
 			return errorMalformedTemplate
 		}
 	}

api/utils_html.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"golang.org/x/net/html"
+	"net/http"
+)
+
+func htmlTitleRecurse(h *html.Node) string {
+	if h == nil || h.FirstChild == nil {
+		return ""
+	}
+
+	if h.Type == html.ElementNode && h.Data == "title" {
+		return h.FirstChild.Data
+	}
+
+	for c := h.FirstChild; c != nil; c = c.NextSibling {
+		res := htmlTitleRecurse(c)
+		if res != "" {
+			return res
+		}
+	}
+
+	return ""
+}
+
+func htmlTitleGet(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	h, err := html.Parse(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return htmlTitleRecurse(h), nil
+}

api/utils_http.go

@@ -43,3 +43,16 @@ func bodyMarshal(w http.ResponseWriter, x map[string]interface{}) error {
 	w.Write(resp)
 	return nil
 }
+
+func getIp(r *http.Request) string {
+	ip := r.RemoteAddr
+	if r.Header.Get("X-Forwarded-For") != "" {
+		ip = r.Header.Get("X-Forwarded-For")
+	}
+
+	return ip
+}
+
+func getUserAgent(r *http.Request) string {
+	return r.Header.Get("User-Agent")
+}

api/utils_misc.go

@@ -10,6 +10,16 @@ func concat(a bytes.Buffer, b bytes.Buffer) []byte {
 	return append(a.Bytes(), b.Bytes()...)
 }
 
+func nameFromEmail(email string) string {
+	for i, c := range email {
+		if c == '@' {
+			return email[:i]
+		}
+	}
+
+	return email
+}
+
 func exitIfError(err error) {
 	if err != nil {
 		fmt.Printf("fatal error: %v\n", err)

api/version.go

@@ -6,21 +6,24 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"time"
 )
 
+func versionPrint() error {
+	logger.Infof("starting Commento %s", version)
+	return nil
+}
+
 func versionCheckStart() error {
 	go func() {
 		printedError := false
 		errorCount := 0
+		latestSeen := ""
 
 		for {
 			time.Sleep(5 * time.Minute)
 
 			data := url.Values{
-				"origin":  {os.Getenv("ORIGIN")},
-				"edition": {edition},
 				"version": {version},
 			}
 
@@ -65,8 +68,9 @@ func versionCheckStart() error {
 				continue
 			}
 
-			if r.NewUpdate {
+			if r.NewUpdate && r.Latest != latestSeen {
 				logger.Infof("New update available! Latest version: %s", r.Latest)
+				latestSeen = r.Latest
 			}
 
 			errorCount = 0

db/20181007230906-store-version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS config (
+  version                  TEXT          NOT NULL
+);
+
+INSERT INTO
+config (version)
+VALUES ('v1.1.3');

db/20181007231407-v1.1.4.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.2.0';

db/20181218183803-sticky-comments.sql

@@ -0,0 +1,2 @@
+ALTER TABLE pages
+  ADD stickyCommentHex TEXT NOT NULL DEFAULT 'none';

db/20181228114101-v1.4.0.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.4.0';

db/20181228114101-v1.4.1.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.4.1';

db/20190122235525-anonymous-moderation-default.sql

@@ -0,0 +1,4 @@
+-- Allow the owner to change whether anonymous comments are put into moderation by default.
+
+ALTER TABLE domains
+  ADD COLUMN moderateAllAnonymous BOOLEAN DEFAULT true;

db/20190123002724-v1.4.2.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.4.2';

db/20190131002240-export.sql

@@ -0,0 +1,8 @@
+-- add export feature
+
+CREATE TABLE IF NOT EXISTS exports (
+  exportHex TEXT NOT NULL UNIQUE PRIMARY KEY,
+  binData BYTEA NOT NULL,
+  domain TEXT NOT NULL,
+  creationDate TIMESTAMP NOT NULL
+);

db/20190204180609-v1.5.0.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.5.0';

db/20190213033530-email-notifications.sql

@@ -0,0 +1,38 @@
+-- Email notifications
+-- There are two kinds of email notifications: those sent to domain moderators
+-- and those sent to commenters. Domain owners can choose to subscribe their
+-- moderators to all comments, those pending moderation, or no emails. Each
+-- moderator can independently opt out of these emails, of course. Commenters,
+-- on the other, can choose to opt into reply notifications by email.
+
+-- TODO: daily and weekly digests instead of just batched real-time emails?
+
+-- TODO: more granular options to unsubscribe from emails for particular
+-- domains can be provided - add unsubscribedReplyDomains []TEXT and
+-- unsubscribedModeratorDomains []TEXT to emails table?
+
+-- Each address has a cooldown period so that emails aren't sent within 10
+-- minutes of each other. Why is this a separate table instead of another
+-- column on commenters/owners? Because there may be some mods that haven't
+-- logged in to create a row in the commenter table.
+CREATE TABLE IF NOT EXISTS emails (
+  email TEXT NOT NULL UNIQUE PRIMARY KEY,
+  unsubscribeSecretHex TEXT NOT NULL UNIQUE,
+  lastEmailNotificationDate TIMESTAMP NOT NULL,
+  pendingEmails INTEGER NOT NULL DEFAULT 0,
+  sendReplyNotifications BOOLEAN NOT NULL DEFAULT false,
+  sendModeratorNotifications BOOLEAN NOT NULL DEFAULT true
+);
+
+CREATE INDEX IF NOT EXISTS unsubscribeSecretHexIndex ON emails(unsubscribeSecretHex);
+
+-- Which comments should be sent?
+-- Possible values: all, pending-moderation, none
+-- Default to pending-moderation because this is critical. If the user forgets
+-- to moderate, some comments will never see the light of day.
+ALTER TABLE domains
+  ADD COLUMN emailNotificationPolicy TEXT DEFAULT 'pending-moderation';
+
+-- Each page now needs to store the title of the page.
+ALTER TABLE pages
+  ADD COLUMN title TEXT DEFAULT '';

db/20190218173502-v1.6.0.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.6.0';

db/20190219001130-v1.6.2.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.6.0';

db/20190418210855-configurable-auth.sql

@@ -0,0 +1,16 @@
+-- Make all login providers optional (but enabled by default)
+
+ALTER TABLE domains
+  ADD commentoProvider BOOLEAN NOT NULL DEFAULT true;
+
+ALTER TABLE domains
+  ADD googleProvider BOOLEAN NOT NULL DEFAULT true;
+
+ALTER TABLE domains
+  ADD twitterProvider BOOLEAN NOT NULL DEFAULT true;
+
+ALTER TABLE domains
+  ADD githubProvider BOOLEAN NOT NULL DEFAULT true;
+
+ALTER TABLE domains
+  ADD gitlabProvider BOOLEAN NOT NULL DEFAULT true;

db/20190420181913-sso.sql

@@ -0,0 +1,10 @@
+-- Single Sign-On (SSO)
+
+ALTER TABLE domains
+  ADD ssoProvider BOOLEAN NOT NULL DEFAULT false;
+
+ALTER TABLE domains
+  ADD ssoSecret TEXT NOT NULL DEFAULT '';
+
+ALTER TABLE domains
+  ADD ssoUrl TEXT NOT NULL DEFAULT '';

db/20190420231030-sso-tokens.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS ssoTokens (
+  token                    TEXT          NOT NULL  UNIQUE  PRIMARY KEY      ,
+  domain                   TEXT          NOT NULL                           ,
+  commenterToken           TEXT          NOT NULL                           ,
+  creationDate             TIMESTAMP     NOT NULL
+);

db/20190501201032-v1.7.0.sql

@@ -0,0 +1,2 @@
+UPDATE config
+SET version = 'v1.7.0';

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   server:
-    image: registry.gitlab.com/commento/commento-ce
+    image: registry.gitlab.com/commento/commento
     ports:
       - 8080:8080
     environment:
@@ -19,12 +19,10 @@ services:
       POSTGRES_DB: commento
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
-    ports:
-      - 5432:5432
     networks:
       - db_network
     volumes:
-      - postgres_data_volume:/var/lib/postgres
+      - postgres_data_volume:/var/lib/postgresql/data
 
 networks:
   db_network: