Add wayback to Telegraph support

wabarc · Jan 28, 2021 · ce1381f · ce1381f
1 parent f5036f8
commit ce1381f
Show file tree

Hide file tree

Showing 14 changed files with 191 additions and 128 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.0] - 2021-01-28
+
+### Add
+- Add wayback to Telegraph support.
+
+### Changed
+- Using `/etc/tor/torrc` for Tor Hidden Service via the `WAYBACK_TORRC` environment variable
+
+### Fixed
+- Minor bugfixs.
+
 ## [0.5.6] - 2021-01-24
 
 ### Changed

diff --git a/README.md b/README.md
@@ -1,20 +1,36 @@
 # Wayback
 
+[![LICENSE](https://img.shields.io/github/license/wabarc/wayback.svg?color=green)](https://github.com/wabarc/wayback/blob/main/LICENSE)
 [![Go Report Card](https://goreportcard.com/badge/github.com/wabarc/wayback)](https://goreportcard.com/report/github.com/wabarc/wayback)
 [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/wabarc/wayback/Go?color=brightgreen)](https://github.com/wabarc/wayback/actions)
+![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/wabarc/wayback)
 [![Releases](https://img.shields.io/github/v/release/wabarc/wayback.svg?include_prereleases&color=blue)](https://github.com/wabarc/wayback/releases)
-[![LICENSE](https://img.shields.io/github/license/wabarc/wayback.svg?color=green)](https://github.com/wabarc/wayback/blob/main/LICENSE)
-[![Docker Automated build](https://img.shields.io/docker/automated/wabarc/wayback)](https://hub.docker.com/r/wabarc/wayback)
-[![wayback](https://snapcraft.io/wayback/badge.svg)](https://snapcraft.io/wayback)
+[![Docker Automated build](https://github.com/wabarc/wayback/workflows/Docker/badge.svg)](https://hub.docker.com/r/wabarc/wayback)
+[![Snapcraft](https://github.com/wabarc/wayback/workflows/Snapcraft/badge.svg)](https://snapcraft.io/wayback)
+
+Wayback is a tool that supports running as a command-line tool and docker container, purpose to snapshot webpage to time capsules.
 
-`wabarc/wayback` is a tool that supports running as a command-line tool and docker container, purpose to snapshot webpage to time capsules.
+## Feature
+
+- CLI tool
+- Interactive with Telegram bot
+- Serve as Tor Hidden Service or local web entry
+- Wayback to Internet Archive, archive.today, IPFS, etc
 
 ## Installation
 
+From source:
+
 ```sh
 $ go get -u github.com/wabarc/wayback/cmd/wayback
 ```
 
+From [GoBinaries](https://gobinaries.com/):
+
+```sh
+$ curl -sf https://gobinaries.com/wabarc/wayback/cmd/wayback | sh
+```
+
 Using [Snapcraft](https://snapcraft.io/wayback) (on GNU/Linux)
 
 ```sh
@@ -23,13 +39,6 @@ $ sudo snap install wayback
 
 See more on [releases](https://github.com/wabarc/wayback/releases).
 
-## Feature
-
-- CLI tool
-- Interactive with telegram bot
-- Serve as Tor hidden service
-- Wayback to Internet Archive, archive.today, IPFS, etc
-
 ## Usage
 
 ### Command line
@@ -61,9 +70,10 @@ Flags:
   -m, --ipfs-mode string   IPFS mode. (default "pinner")
   -p, --ipfs-port uint     IPFS daemon port. (default 5001)
       --is                 Wayback webpages to Archive Today.
+      --ph                 Wayback webpages to Telegraph. (default false)
   -t, --token string       Telegram Bot API Token.
-      --tor                Snapshot webpage via Tor proxy.
-      --tor-key string     The private key for Tor service.
+      --tor                Snapshot webpage via Tor anonymity network.
+      --tor-key string     The private key for Tor Hidden Service.
   -v, --version            version for wayback
 ```
 
@@ -132,95 +142,28 @@ $ wayback -d telegram -t YOUT-BOT-TOKEN -d web
 
 #### Configuration Parameters
 
-Wayback can use command flags and environment variables.
-
-##### `DEBUG`
-
-> Toggle debug mode (display all logging).
->
-> *Disabled by default.*
-
-##### `LOG_TIME`
-
-> Display the date and time in log messages.
->
-> *default: true*
-
-<details>
-<summary>more configurations</summary>
-
-##### `WAYBACK_IPFS_HOST`
-
-> IPFS daemon service's host, do not require, unless enable ipfs.
->
-> *default "127.0.0.1". (same as flag --ipfs-host).*
-
-##### `WAYBACK_IPFS_PORT`
-
-> IPFS daemon port.
->
-> *default 5001. (same as flag --ipfs-port).*
-
-##### `WAYBACK_IPFS_MODE`
-
-> IPFS mode for preserve webpage.
->
-> *default: "pinner". (same as flag --ipfs-mode).*
-
-##### `WAYBACK_USE_TOR`
-
-> Snapshot webpage via Tor proxy. (same as flag --tor).
->
-> *default: false*
-
-##### `WAYBACK_ENABLE_IA`
-
-> Enable Internet Archive.
->
-> *default: true*
-
-##### `WAYBACK_ENABLE_IS`
-
-> Enable Archive Today (archive.is).
->
-> *default: true*
-
-##### `WAYBACK_ENABLE_IP`
-
-> Enable IPFS.
->
-> *default: false*
-
-##### `WAYBACK_TELEGRAM_TOKEN`
-
-> Telegram Bot API Token.
->
-> *default: ""* (same as flag --token).
-
-##### `WAYBACK_TELEGRAM_CHANNEL`
-
-> The chatid which publish message.
->
-> *default: ""* (same as flag --chatid).
-
-##### `WAYBACK_TOR_PRIVKEY`
-
-> The private key for Tor service.
->
-> *default: ""* (same as flag --tor-key).
-
-##### `WAYBACK_TOR_LOCAL_PORT`
-
-> Local port of Tor service.
->
-> *default: 0*
-
-##### `WAYBACK_TOR_REMOTE_PORTS`
-
-> Remote ports of Tor hidden service, e.g. `WAYBACK_TOR_REMOTE_PORTS=80,81`.
->
-> *default: 80*
-</details>
+You can specify configuration options either via command flags or via environment variables, an overview of all options below.
+
+
+| Flags               | Environment Variable       | Default     | Description                                                  |
+| ------------------- | -------------------------- | ----------- | ------------------------------------------------------------ |
+| `--debug`           | `DEBUG`                    | `false`     | Enable debug mode                                            |
+| -                   | `LOG_TIME`                 | `true`      | Display the date and time in log messages                    |
+| `-d`, `--daemon`    | -                          | -           | Run as daemon service, e.g. `telegram`, `web`                |
+| `--ia`              | `WAYBACK_ENABLE_IA`        | `true`      | Wayback webpages to **Internet Archive**                     |
+| `--is`              | `WAYBACK_ENABLE_IS`        | `true`      | Wayback webpages to **Archive Today**                        |
+| `--ip`              | `WAYBACK_ENABLE_IP`        | `false`     | Wayback webpages to **IPFS**                                 |
+| `--ph`              | `WAYBACK_ENABLE_PH`        | `false`     | Wayback webpages to **[Telegra.ph](https://telegra.ph)**, required Chrome/Chromium |
+| `--ipfs-host`       | `WAYBACK_IPFS_HOST`        | `127.0.0.1` | IPFS daemon service host                                     |
+| `-p`, `--ipfs-port` | `WAYBACK_IPFS_PORT`        | `5001`      | IPFS daemon service port                                     |
+| `-m`, `--ipfs-mode` | `WAYBACK_IPFS_MODE`        | `pinner`    | IPFS mode for preserve webpage, e.g. `daemon`, `pinner`      |
+| `-t`, `--token`     | `WAYBACK_TELEGRAM_TOKEN`   | -           | Telegram Bot API Token                                       |
+| `-c`, `--chatid`    | `WAYBACK_TELEGRAM_CHANNEL` | -           | The **Telegram Channel** name for publish archived result    |
+| `--tor`             | `WAYBACK_USE_TOR`          | `false`     | Snapshot webpage via Tor anonymity network                   |
+| `--tor-key`         | `WAYBACK_TOR_PRIVKEY`      | -           | The private key for Tor Hidden Service                       |
+| -                   | `WAYBACK_TOR_LOCAL_PORT`   | -           | Local port for Tor Hidden Service, also support for a **reverse proxy** |
+| -                   | `WAYBACK_TOR_REMOTE_PORTS` | `80`        | Remote ports for Tor Hidden Service, e.g. `WAYBACK_TOR_REMOTE_PORTS=80,81` |
+| -                   | `WAYBACK_TORRC`            | `/etc/tor/torrc` | Using `torrc` for Tor Hidden Service |
 
 ### Docker/Podman
 

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
@@ -17,14 +17,32 @@ RUN sh ./build/binary.sh $TARGETPLATFORM \
 ############################
 # STEP 2 build a small image
 ############################
-FROM alpine:3.12
+FROM alpine:3.13
 
 LABEL maintainer "WaybackBot <[email protected]>"
-COPY --from=builder /wayback /usr/local/bin
-RUN apk update && apk add ca-certificates tor
-RUN mv /etc/tor/torrc.sample /etc/tor/torrc
-RUN echo 'ExcludeNodes {cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}' >> /etc/tor/torrc
-RUN echo 'ExcludeExitNodes {cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}' >> /etc/tor/torrc
-RUN echo 'StrictNodes 1' >> /etc/tor/torrc
 
-RUN rm -rf /var/cache/apk/*
+ARG TOR_EXCLUDE_NODE="{cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}"
+
+ENV BASE_DIR /wayback
+ENV PUSER wayback
+ENV PGROUP wayback
+
+WORKDIR $BASE_DIR
+
+RUN set -eux; \
+    addgroup --system ${PGROUP}; \
+    adduser --system --no-create-home --disabled-password \
+    --gecos '' --home ${BASE_DIR} --ingroup ${PGROUP} ${PUSER}; \
+    chown -R ${PUSER}:${PGROUP} "${BASE_DIR}"; \
+    chmod -R g+w "${BASE_DIR}"
+
+COPY --from=builder /wayback /usr/local/bin
+RUN apk update; \
+    apk add --no-cache ca-certificates tor; \
+    rm -rf /var/cache/apk/*; \
+    \
+    mv /etc/tor/torrc.sample /etc/tor/torrc; \
+    echo "ExcludeNodes ${TOR_EXCLUDE_NODE}" >> /etc/tor/torrc; \
+    echo "ExcludeExitNodes ${TOR_EXCLUDE_NODE}" >> /etc/tor/torrc; \
+    echo 'StrictNodes 1' >> /etc/tor/torrc; \
+    #echo 'User tor' >> /etc/tor/torrc
diff --git a/cmd/wayback/main.go b/cmd/wayback/main.go
@@ -16,6 +16,7 @@ var (
 	ia bool
 	is bool
 	ip bool
+	ph bool
 
 	daemon []string
 
@@ -56,16 +57,17 @@ func init() {
 	rootCmd.Flags().BoolVarP(&ia, "ia", "", false, "Wayback webpages to Internet Archive.")
 	rootCmd.Flags().BoolVarP(&is, "is", "", false, "Wayback webpages to Archive Today.")
 	rootCmd.Flags().BoolVarP(&ip, "ip", "", false, "Wayback webpages to IPFS. (default false)")
+	rootCmd.Flags().BoolVarP(&ph, "ph", "", false, "Wayback webpages to Telegraph. (default false)")
 	rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, e.g. telegram, web")
 	rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs.")
 	rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port.")
 	rootCmd.Flags().StringVarP(&mode, "ipfs-mode", "m", "pinner", "IPFS mode.")
-	rootCmd.Flags().BoolVarP(&tor, "tor", "", false, "Snapshot webpage via Tor proxy.")
+	rootCmd.Flags().BoolVarP(&tor, "tor", "", false, "Snapshot webpage via Tor anonymity network.")
 
 	rootCmd.Flags().StringVarP(&token, "token", "t", "", "Telegram Bot API Token.")
 	rootCmd.Flags().StringVarP(&chatid, "chatid", "c", "", "Telegram channel id.")
 	rootCmd.Flags().BoolVarP(&debug, "debug", "", false, "Enable debug mode. (default false)")
-	rootCmd.Flags().StringVarP(&torKey, "tor-key", "", "", "The private key for Tor service.")
+	rootCmd.Flags().StringVarP(&torKey, "tor-key", "", "", "The private key for Tor Hidden Service.")
 }
 
 func checkRequiredFlags(cmd *cobra.Command, args []string) error {
@@ -93,6 +95,9 @@ func checkRequiredFlags(cmd *cobra.Command, args []string) error {
 func setToEnv(cmd *cobra.Command) {
 	flags := cmd.Flags()
 
+	if flags.Changed("debug") {
+		os.Setenv("DEBUG", fmt.Sprint(debug))
+	}
 	if flags.Changed("ia") {
 		os.Setenv("WAYBACK_ENABLE_IA", fmt.Sprint(ia))
 	}
@@ -102,6 +107,9 @@ func setToEnv(cmd *cobra.Command) {
 	if flags.Changed("ip") {
 		os.Setenv("WAYBACK_ENABLE_IP", fmt.Sprint(ip))
 	}
+	if flags.Changed("ph") {
+		os.Setenv("WAYBACK_ENABLE_PH", fmt.Sprint(ph))
+	}
 	if flags.Changed("token") {
 		os.Setenv("WAYBACK_TELEGRAM_TOKEN", token)
 	}
@@ -126,7 +134,7 @@ func setToEnv(cmd *cobra.Command) {
 }
 
 func handle(cmd *cobra.Command, args []string) {
-	if !ia && !is && !ip {
+	if !ia && !is && !ip && !ph {
 		ia, is = true, true
 		os.Setenv("WAYBACK_ENABLE_IA", "true")
 		os.Setenv("WAYBACK_ENABLE_IS", "true")

diff --git a/cmd/wayback/serve.go b/cmd/wayback/serve.go
@@ -27,17 +27,16 @@ func serve(cmd *cobra.Command, opts *config.Options, args []string) {
 }
 
 func (srv *service) run(ctx context.Context, opts *config.Options) *service {
-	telegram := telegram.New(opts)
-	tor := anonymity.New(opts)
-
 	srv.errCh = make(chan error, len(daemon))
 	for _, s := range daemon {
 		switch s {
 		case "telegram":
+			telegram := telegram.New(opts)
 			go func(errCh chan error) {
 				errCh <- telegram.Serve(ctx)
 			}(srv.errCh)
 		case "web":
+			tor := anonymity.New(opts)
 			go func(errCh chan error) {
 				errCh <- tor.Serve(ctx)
 			}(srv.errCh)

diff --git a/config/config.go b/config/config.go
@@ -11,6 +11,7 @@ const (
 	SLOT_IA = "ia"
 	SLOT_IS = "is"
 	SLOT_IP = "ip"
+	SLOT_PH = "ph"
 )
 
 // SlotName returns the descriptions of the wayback service.
@@ -19,6 +20,7 @@ func SlotName(s string) string {
 		SLOT_IA: "Internet Archive",
 		SLOT_IS: "archive.today",
 		SLOT_IP: "IPFS",
+		SLOT_PH: "Telegraph",
 	}
 
 	return slots[s]

diff --git a/config/config_test.go b/config/config_test.go
@@ -178,6 +178,7 @@ func TestEnableSlots(t *testing.T) {
 	os.Setenv("WAYBACK_ENABLE_IA", "true")
 	os.Setenv("WAYBACK_ENABLE_IS", "true")
 	os.Setenv("WAYBACK_ENABLE_IP", "true")
+	os.Setenv("WAYBACK_ENABLE_PH", "true")
 
 	parser := NewParser()
 	opts, err := parser.ParseEnvironmentVariables()
@@ -189,10 +190,11 @@ func TestEnableSlots(t *testing.T) {
 		SLOT_IA: true,
 		SLOT_IS: true,
 		SLOT_IP: true,
+		SLOT_PH: true,
 	}
 	got := opts.Slots()
 
-	if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] {
+	if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] {
 		t.Fatalf(`Unexpected over Tor, got %v instead of %v`, got, expected)
 	}
 }