diff --git a/CHANGELOG.md b/CHANGELOG.md index b2dcf313..b2118816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.0] - 2021-01-28 + +### Add +- Add wayback to Telegraph support. + +### Changed +- Using `/etc/tor/torrc` for Tor Hidden Service via the `WAYBACK_TORRC` environment variable + +### Fixed +- Minor bugfixs. + ## [0.5.6] - 2021-01-24 ### Changed diff --git a/README.md b/README.md index d5da1019..e45ac05e 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,36 @@ # Wayback +[![LICENSE](https://img.shields.io/github/license/wabarc/wayback.svg?color=green)](https://github.com/wabarc/wayback/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/wabarc/wayback)](https://goreportcard.com/report/github.com/wabarc/wayback) [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/wabarc/wayback/Go?color=brightgreen)](https://github.com/wabarc/wayback/actions) +![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/wabarc/wayback) [![Releases](https://img.shields.io/github/v/release/wabarc/wayback.svg?include_prereleases&color=blue)](https://github.com/wabarc/wayback/releases) -[![LICENSE](https://img.shields.io/github/license/wabarc/wayback.svg?color=green)](https://github.com/wabarc/wayback/blob/main/LICENSE) -[![Docker Automated build](https://img.shields.io/docker/automated/wabarc/wayback)](https://hub.docker.com/r/wabarc/wayback) -[![wayback](https://snapcraft.io/wayback/badge.svg)](https://snapcraft.io/wayback) +[![Docker Automated build](https://github.com/wabarc/wayback/workflows/Docker/badge.svg)](https://hub.docker.com/r/wabarc/wayback) +[![Snapcraft](https://github.com/wabarc/wayback/workflows/Snapcraft/badge.svg)](https://snapcraft.io/wayback) + +Wayback is a tool that supports running as a command-line tool and docker container, purpose to snapshot webpage to time capsules. -`wabarc/wayback` is a tool that supports running as a command-line tool and docker container, purpose to snapshot webpage to time capsules. +## Feature + +- CLI tool +- Interactive with Telegram bot +- Serve as Tor Hidden Service or local web entry +- Wayback to Internet Archive, archive.today, IPFS, etc ## Installation +From source: + ```sh $ go get -u github.com/wabarc/wayback/cmd/wayback ``` +From [GoBinaries](https://gobinaries.com/): + +```sh +$ curl -sf https://gobinaries.com/wabarc/wayback/cmd/wayback | sh +``` + Using [Snapcraft](https://snapcraft.io/wayback) (on GNU/Linux) ```sh @@ -23,13 +39,6 @@ $ sudo snap install wayback See more on [releases](https://github.com/wabarc/wayback/releases). -## Feature - -- CLI tool -- Interactive with telegram bot -- Serve as Tor hidden service -- Wayback to Internet Archive, archive.today, IPFS, etc - ## Usage ### Command line @@ -61,9 +70,10 @@ Flags: -m, --ipfs-mode string IPFS mode. (default "pinner") -p, --ipfs-port uint IPFS daemon port. (default 5001) --is Wayback webpages to Archive Today. + --ph Wayback webpages to Telegraph. (default false) -t, --token string Telegram Bot API Token. - --tor Snapshot webpage via Tor proxy. - --tor-key string The private key for Tor service. + --tor Snapshot webpage via Tor anonymity network. + --tor-key string The private key for Tor Hidden Service. -v, --version version for wayback ``` @@ -132,95 +142,28 @@ $ wayback -d telegram -t YOUT-BOT-TOKEN -d web #### Configuration Parameters -Wayback can use command flags and environment variables. - -##### `DEBUG` - -> Toggle debug mode (display all logging). -> -> *Disabled by default.* - -##### `LOG_TIME` - -> Display the date and time in log messages. -> -> *default: true* - -
-more configurations - -##### `WAYBACK_IPFS_HOST` - -> IPFS daemon service's host, do not require, unless enable ipfs. -> -> *default "127.0.0.1". (same as flag --ipfs-host).* - -##### `WAYBACK_IPFS_PORT` - -> IPFS daemon port. -> -> *default 5001. (same as flag --ipfs-port).* - -##### `WAYBACK_IPFS_MODE` - -> IPFS mode for preserve webpage. -> -> *default: "pinner". (same as flag --ipfs-mode).* - -##### `WAYBACK_USE_TOR` - -> Snapshot webpage via Tor proxy. (same as flag --tor). -> -> *default: false* - -##### `WAYBACK_ENABLE_IA` - -> Enable Internet Archive. -> -> *default: true* - -##### `WAYBACK_ENABLE_IS` - -> Enable Archive Today (archive.is). -> -> *default: true* - -##### `WAYBACK_ENABLE_IP` - -> Enable IPFS. -> -> *default: false* - -##### `WAYBACK_TELEGRAM_TOKEN` - -> Telegram Bot API Token. -> -> *default: ""* (same as flag --token). - -##### `WAYBACK_TELEGRAM_CHANNEL` - -> The chatid which publish message. -> -> *default: ""* (same as flag --chatid). - -##### `WAYBACK_TOR_PRIVKEY` - -> The private key for Tor service. -> -> *default: ""* (same as flag --tor-key). - -##### `WAYBACK_TOR_LOCAL_PORT` - -> Local port of Tor service. -> -> *default: 0* - -##### `WAYBACK_TOR_REMOTE_PORTS` - -> Remote ports of Tor hidden service, e.g. `WAYBACK_TOR_REMOTE_PORTS=80,81`. -> -> *default: 80* -
+You can specify configuration options either via command flags or via environment variables, an overview of all options below. + + +| Flags | Environment Variable | Default | Description | +| ------------------- | -------------------------- | ----------- | ------------------------------------------------------------ | +| `--debug` | `DEBUG` | `false` | Enable debug mode | +| - | `LOG_TIME` | `true` | Display the date and time in log messages | +| `-d`, `--daemon` | - | - | Run as daemon service, e.g. `telegram`, `web` | +| `--ia` | `WAYBACK_ENABLE_IA` | `true` | Wayback webpages to **Internet Archive** | +| `--is` | `WAYBACK_ENABLE_IS` | `true` | Wayback webpages to **Archive Today** | +| `--ip` | `WAYBACK_ENABLE_IP` | `false` | Wayback webpages to **IPFS** | +| `--ph` | `WAYBACK_ENABLE_PH` | `false` | Wayback webpages to **[Telegra.ph](https://telegra.ph)**, required Chrome/Chromium | +| `--ipfs-host` | `WAYBACK_IPFS_HOST` | `127.0.0.1` | IPFS daemon service host | +| `-p`, `--ipfs-port` | `WAYBACK_IPFS_PORT` | `5001` | IPFS daemon service port | +| `-m`, `--ipfs-mode` | `WAYBACK_IPFS_MODE` | `pinner` | IPFS mode for preserve webpage, e.g. `daemon`, `pinner` | +| `-t`, `--token` | `WAYBACK_TELEGRAM_TOKEN` | - | Telegram Bot API Token | +| `-c`, `--chatid` | `WAYBACK_TELEGRAM_CHANNEL` | - | The **Telegram Channel** name for publish archived result | +| `--tor` | `WAYBACK_USE_TOR` | `false` | Snapshot webpage via Tor anonymity network | +| `--tor-key` | `WAYBACK_TOR_PRIVKEY` | - | The private key for Tor Hidden Service | +| - | `WAYBACK_TOR_LOCAL_PORT` | - | Local port for Tor Hidden Service, also support for a **reverse proxy** | +| - | `WAYBACK_TOR_REMOTE_PORTS` | `80` | Remote ports for Tor Hidden Service, e.g. `WAYBACK_TOR_REMOTE_PORTS=80,81` | +| - | `WAYBACK_TORRC` | `/etc/tor/torrc` | Using `torrc` for Tor Hidden Service | ### Docker/Podman diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index dde16985..264ed2de 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -17,14 +17,32 @@ RUN sh ./build/binary.sh $TARGETPLATFORM \ ############################ # STEP 2 build a small image ############################ -FROM alpine:3.12 +FROM alpine:3.13 LABEL maintainer "WaybackBot " -COPY --from=builder /wayback /usr/local/bin -RUN apk update && apk add ca-certificates tor -RUN mv /etc/tor/torrc.sample /etc/tor/torrc -RUN echo 'ExcludeNodes {cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}' >> /etc/tor/torrc -RUN echo 'ExcludeExitNodes {cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}' >> /etc/tor/torrc -RUN echo 'StrictNodes 1' >> /etc/tor/torrc -RUN rm -rf /var/cache/apk/* +ARG TOR_EXCLUDE_NODE="{cn},{hk},{mo},{sg},{th},{pk},{by},{ru},{ir},{sy},{vn},{ph},{my},{cu},{us},{au},{ca},{nz},{gb},{fr}" + +ENV BASE_DIR /wayback +ENV PUSER wayback +ENV PGROUP wayback + +WORKDIR $BASE_DIR + +RUN set -eux; \ + addgroup --system ${PGROUP}; \ + adduser --system --no-create-home --disabled-password \ + --gecos '' --home ${BASE_DIR} --ingroup ${PGROUP} ${PUSER}; \ + chown -R ${PUSER}:${PGROUP} "${BASE_DIR}"; \ + chmod -R g+w "${BASE_DIR}" + +COPY --from=builder /wayback /usr/local/bin +RUN apk update; \ + apk add --no-cache ca-certificates tor; \ + rm -rf /var/cache/apk/*; \ + \ + mv /etc/tor/torrc.sample /etc/tor/torrc; \ + echo "ExcludeNodes ${TOR_EXCLUDE_NODE}" >> /etc/tor/torrc; \ + echo "ExcludeExitNodes ${TOR_EXCLUDE_NODE}" >> /etc/tor/torrc; \ + echo 'StrictNodes 1' >> /etc/tor/torrc; \ + #echo 'User tor' >> /etc/tor/torrc diff --git a/cmd/wayback/main.go b/cmd/wayback/main.go index 0f02e5a5..e3f4f707 100644 --- a/cmd/wayback/main.go +++ b/cmd/wayback/main.go @@ -16,6 +16,7 @@ var ( ia bool is bool ip bool + ph bool daemon []string @@ -56,16 +57,17 @@ func init() { rootCmd.Flags().BoolVarP(&ia, "ia", "", false, "Wayback webpages to Internet Archive.") rootCmd.Flags().BoolVarP(&is, "is", "", false, "Wayback webpages to Archive Today.") rootCmd.Flags().BoolVarP(&ip, "ip", "", false, "Wayback webpages to IPFS. (default false)") + rootCmd.Flags().BoolVarP(&ph, "ph", "", false, "Wayback webpages to Telegraph. (default false)") rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, e.g. telegram, web") rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs.") rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port.") rootCmd.Flags().StringVarP(&mode, "ipfs-mode", "m", "pinner", "IPFS mode.") - rootCmd.Flags().BoolVarP(&tor, "tor", "", false, "Snapshot webpage via Tor proxy.") + rootCmd.Flags().BoolVarP(&tor, "tor", "", false, "Snapshot webpage via Tor anonymity network.") rootCmd.Flags().StringVarP(&token, "token", "t", "", "Telegram Bot API Token.") rootCmd.Flags().StringVarP(&chatid, "chatid", "c", "", "Telegram channel id.") rootCmd.Flags().BoolVarP(&debug, "debug", "", false, "Enable debug mode. (default false)") - rootCmd.Flags().StringVarP(&torKey, "tor-key", "", "", "The private key for Tor service.") + rootCmd.Flags().StringVarP(&torKey, "tor-key", "", "", "The private key for Tor Hidden Service.") } func checkRequiredFlags(cmd *cobra.Command, args []string) error { @@ -93,6 +95,9 @@ func checkRequiredFlags(cmd *cobra.Command, args []string) error { func setToEnv(cmd *cobra.Command) { flags := cmd.Flags() + if flags.Changed("debug") { + os.Setenv("DEBUG", fmt.Sprint(debug)) + } if flags.Changed("ia") { os.Setenv("WAYBACK_ENABLE_IA", fmt.Sprint(ia)) } @@ -102,6 +107,9 @@ func setToEnv(cmd *cobra.Command) { if flags.Changed("ip") { os.Setenv("WAYBACK_ENABLE_IP", fmt.Sprint(ip)) } + if flags.Changed("ph") { + os.Setenv("WAYBACK_ENABLE_PH", fmt.Sprint(ph)) + } if flags.Changed("token") { os.Setenv("WAYBACK_TELEGRAM_TOKEN", token) } @@ -126,7 +134,7 @@ func setToEnv(cmd *cobra.Command) { } func handle(cmd *cobra.Command, args []string) { - if !ia && !is && !ip { + if !ia && !is && !ip && !ph { ia, is = true, true os.Setenv("WAYBACK_ENABLE_IA", "true") os.Setenv("WAYBACK_ENABLE_IS", "true") diff --git a/cmd/wayback/serve.go b/cmd/wayback/serve.go index d1fbe9a8..2bc3015c 100644 --- a/cmd/wayback/serve.go +++ b/cmd/wayback/serve.go @@ -27,17 +27,16 @@ func serve(cmd *cobra.Command, opts *config.Options, args []string) { } func (srv *service) run(ctx context.Context, opts *config.Options) *service { - telegram := telegram.New(opts) - tor := anonymity.New(opts) - srv.errCh = make(chan error, len(daemon)) for _, s := range daemon { switch s { case "telegram": + telegram := telegram.New(opts) go func(errCh chan error) { errCh <- telegram.Serve(ctx) }(srv.errCh) case "web": + tor := anonymity.New(opts) go func(errCh chan error) { errCh <- tor.Serve(ctx) }(srv.errCh) diff --git a/config/config.go b/config/config.go index 2ba660e1..58bd451d 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ const ( SLOT_IA = "ia" SLOT_IS = "is" SLOT_IP = "ip" + SLOT_PH = "ph" ) // SlotName returns the descriptions of the wayback service. @@ -19,6 +20,7 @@ func SlotName(s string) string { SLOT_IA: "Internet Archive", SLOT_IS: "archive.today", SLOT_IP: "IPFS", + SLOT_PH: "Telegraph", } return slots[s] diff --git a/config/config_test.go b/config/config_test.go index b76b94f2..41862b63 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -178,6 +178,7 @@ func TestEnableSlots(t *testing.T) { os.Setenv("WAYBACK_ENABLE_IA", "true") os.Setenv("WAYBACK_ENABLE_IS", "true") os.Setenv("WAYBACK_ENABLE_IP", "true") + os.Setenv("WAYBACK_ENABLE_PH", "true") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -189,10 +190,11 @@ func TestEnableSlots(t *testing.T) { SLOT_IA: true, SLOT_IS: true, SLOT_IP: true, + SLOT_PH: true, } got := opts.Slots() - if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] { + if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] { t.Fatalf(`Unexpected over Tor, got %v instead of %v`, got, expected) } } diff --git a/config/options.go b/config/options.go index 5977d47a..21948015 100644 --- a/config/options.go +++ b/config/options.go @@ -16,12 +16,14 @@ const ( defEnabledIA = false defEnabledIS = false defEnabledIP = false + defEnabledPH = false defTelegramToken = "" defTelegramChannel = "" defTorPrivateKey = "" defTorLocalPort = 0 + defTorrcFile = "/etc/tor/torrc" ) var ( @@ -49,6 +51,7 @@ type slots struct { ia bool is bool ip bool + ph bool } type telegram struct { @@ -61,6 +64,7 @@ type tor struct { localPort int remotePorts []int + torrcFile string } // NewOptions returns Options with default values. @@ -78,6 +82,7 @@ func NewOptions() *Options { SLOT_IA: defEnabledIA, SLOT_IS: defEnabledIS, SLOT_IP: defEnabledIP, + SLOT_PH: defEnabledPH, }, telegram: &telegram{ token: defTelegramToken, @@ -87,6 +92,7 @@ func NewOptions() *Options { pvk: defTorPrivateKey, localPort: defTorLocalPort, remotePorts: defTorRemotePorts, + torrcFile: defTorrcFile, }, } @@ -152,3 +158,8 @@ func (o *Options) TorLocalPort() int { func (o *Options) TorRemotePorts() []int { return o.tor.remotePorts } + +// TorrcFile returns path of the torrc file to set on start Tor Hidden Service. +func (o *Options) TorrcFile() string { + return o.tor.torrcFile +} diff --git a/config/parser.go b/config/parser.go index b3a93977..10366416 100644 --- a/config/parser.go +++ b/config/parser.go @@ -56,6 +56,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.slots[SLOT_IS] = parseBool(val, defEnabledIS) case "WAYBACK_ENABLE_IP": p.opts.slots[SLOT_IP] = parseBool(val, defEnabledIP) + case "WAYBACK_ENABLE_PH": + p.opts.slots[SLOT_PH] = parseBool(val, defEnabledPH) case "WAYBACK_TELEGRAM_TOKEN": p.opts.telegram.token = parseString(val, defTelegramToken) case "WAYBACK_TELEGRAM_CHANNEL": @@ -66,6 +68,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.tor.localPort = parseInt(val, defTorLocalPort) case "WAYBACK_TOR_REMOTE_PORTS": p.opts.tor.remotePorts = parseIntList(val, defTorRemotePorts) + case "WAYBACK_TORRC": + p.opts.tor.torrcFile = parseString(val, defTorrcFile) } } diff --git a/go.mod b/go.mod index 8c90e8d6..732bf0ac 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/cretz/bine v0.1.0 github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible + github.com/google/uuid v1.2.0 // indirect // github.com/ipsn/go-libtor v1.0.329 github.com/libp2p/go-libp2p-core v0.8.0 // indirect github.com/multiformats/go-multiaddr v0.3.1 // indirect @@ -15,13 +16,14 @@ require ( github.com/technoweenie/multipartstreamer v1.0.1 // indirect github.com/wabarc/archive.is v1.1.0 github.com/wabarc/archive.org v1.0.3 + github.com/wabarc/screenshot v0.0.0-20210128074521-26790a41f7bd // indirect + github.com/wabarc/telegra.ph v0.0.0-20210127125351-c2084324e22a github.com/wabarc/wbipfs v0.1.2 github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20210104204734-6f8348627aad // indirect ) replace github.com/go-shiori/obelisk => github.com/wabarc/obelisk v0.0.0-20201203134337-61a4e7973e4c diff --git a/go.sum b/go.sum index db116138..a8d7a229 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,13 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chromedp/cdproto v0.0.0-20210122124816-7a656c010d57/go.mod h1:55pim6Ht4LJKdVLlyFJV/g++HsEA1hQxPbB5JyNdZC0= +github.com/chromedp/cdproto v0.0.0-20210126020047-7ec7357d1463 h1:jLOnhBJ0rDgLdxTpAT6KlbQWcPTdlsxs+9d1RvX9u1w= +github.com/chromedp/cdproto v0.0.0-20210126020047-7ec7357d1463/go.mod h1:55pim6Ht4LJKdVLlyFJV/g++HsEA1hQxPbB5JyNdZC0= +github.com/chromedp/chromedp v0.6.5 h1:hPaDYBpvD2WFicln0ByzV+XRhSOtLgAgsu39O455iWY= +github.com/chromedp/chromedp v0.6.5/go.mod h1:/Q6h52DkrFuvOgmCuR6O3xT5g0bZYoPqjANKBEvQGEY= +github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -81,6 +88,12 @@ github.com/go-shiori/dom v0.0.0-20201011032054-d6b74a54fe52/go.mod h1:aLEd5DGjh1 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs= +github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -104,6 +117,10 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -150,11 +167,15 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kallydev/telegraph-go v1.0.0 h1:JsIlQfYCY4S5QH20rJYSN6A1eY/vVx07oBVaCsGU/nE= +github.com/kallydev/telegraph-go v1.0.0/go.mod h1:vZj7M9HridntSIuQ7D9hgn2idKiA0T1VIblLp6l9uuc= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -180,6 +201,8 @@ github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGN github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -304,10 +327,18 @@ github.com/wabarc/archive.is v1.1.0 h1:aeK4d6YHT5vDAOJEM1v48VfV0B5AYd821xGFqAhFx github.com/wabarc/archive.is v1.1.0/go.mod h1:tO55Az3CB3szcwFBJmaTRDqJVZTLFdSPkMJVatLFg00= github.com/wabarc/archive.org v1.0.3 h1:YiWn7Bn6iiRfApuquYWHXW1dmz571rQ/8LyKngJSEMM= github.com/wabarc/archive.org v1.0.3/go.mod h1:jxYrX2hfBi/Pj8AWkp1VuvmsKUhO0mRpIy3PlYVQ9lk= +github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616 h1:wZ5HtpmZAVUq0Im5Sm92ycJrTeLJk5lB/Kvh55Rd+Ps= +github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8= github.com/wabarc/ipfs-pinner v1.0.1 h1:PaWBDn1X556IyiOVd143XO9PVRjbohppA+JMJQwNBHI= github.com/wabarc/ipfs-pinner v1.0.1/go.mod h1:3EsCzUMiCsqnHr/fbHHj3r8s72CHQIm3JdiOpYyh27E= github.com/wabarc/obelisk v0.0.0-20201203134337-61a4e7973e4c h1:wq9ZNfly9sj4E18zGRkLVysH5pxg4KIkBiMIQGBgPaw= github.com/wabarc/obelisk v0.0.0-20201203134337-61a4e7973e4c/go.mod h1:OlGUo0utWqPMcej0HirsoK2+65Je/ZPFNwasF/O3KYM= +github.com/wabarc/screenshot v0.0.0-20210127123919-7b249b9e1d08 h1:Sykq9OPY9w8yUKDPQklnC0sUxdxNukLbAQJwk2p+Dqs= +github.com/wabarc/screenshot v0.0.0-20210127123919-7b249b9e1d08/go.mod h1:4UF/mOVbZmULZ4qeigCWEXlSp0wS5jtq11EdsbyyN5w= +github.com/wabarc/screenshot v0.0.0-20210128074521-26790a41f7bd h1:8VRBx0HwC3wiosRHFZ/kIlQtHpWwJnIdpXuEPG53qrg= +github.com/wabarc/screenshot v0.0.0-20210128074521-26790a41f7bd/go.mod h1:4UF/mOVbZmULZ4qeigCWEXlSp0wS5jtq11EdsbyyN5w= +github.com/wabarc/telegra.ph v0.0.0-20210127125351-c2084324e22a h1:urtW1KUNmAxzVSSkqqaltEFtNhO6BJNrZvLfvs75NuI= +github.com/wabarc/telegra.ph v0.0.0-20210127125351-c2084324e22a/go.mod h1:3GDRBnpkkHwBTNYe+pkNNT0RGihRX9Vy0kxBRBiuMR8= github.com/wabarc/wbipfs v0.1.2 h1:Q012Xfk3FjNAInSkNWiV3qqtNMQ3UDdZBSojfD+BeMc= github.com/wabarc/wbipfs v0.1.2/go.mod h1:zjeZWvCyZ2MNPx0PbgwgqgbugrWz7C/Abib2fmy54qg= github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= @@ -412,8 +443,9 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad h1:MCsdmFSdEd4UEa5TKS5JztCRHK/WtvNei1edOj5RSRo= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210122093101-04d7465088b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/service/anonymity/tor.go b/service/anonymity/tor.go index 89c2e0a7..6c94ec66 100644 --- a/service/anonymity/tor.go +++ b/service/anonymity/tor.go @@ -66,7 +66,7 @@ func (t *tor) Serve(ctx context.Context) error { verbose := t.opts.HasDebugMode() // startConf := &embedTor.StartConf{ProcessCreator: libtor.Creator, DataDir: "tor-data"} - startConf := &embedTor.StartConf{} + startConf := &embedTor.StartConf{TorrcFile: t.torrc()} if verbose { startConf.DebugWriter = os.Stdout } else { @@ -172,12 +172,12 @@ func (t *tor) archive(ctx context.Context, text string) (tc *template.Collector, continue } wg.Add(1) + logger.Debug("Web: archiving slot: %s", slot) go func(slot string, tc *template.Collector) { defer wg.Done() c := &publish.Collect{} switch slot { case config.SLOT_IA: - logger.Debug("Web: archiving slot: %s", slot) ia := wbrc.IA() slotName := config.SlotName(slot) @@ -188,7 +188,6 @@ func (t *tor) archive(ctx context.Context, text string) (tc *template.Collector, c.Arc = fmt.Sprintf("%s", slotName) c.Dst = ia case config.SLOT_IS: - logger.Debug("Web: archiving slot: %s", slot) is := wbrc.IS() slotName := config.SlotName(slot) @@ -199,7 +198,6 @@ func (t *tor) archive(ctx context.Context, text string) (tc *template.Collector, c.Arc = fmt.Sprintf("%s", slotName) c.Dst = is case config.SLOT_IP: - logger.Debug("Web: archiving slot: %s", slot) ip := wbrc.IP() slotName := config.SlotName(slot) @@ -209,6 +207,16 @@ func (t *tor) archive(ctx context.Context, text string) (tc *template.Collector, // Data for publish c.Arc = fmt.Sprintf("%s", slotName) c.Dst = ip + case config.SLOT_PH: + ph := wbrc.PH() + slotName := config.SlotName(slot) + + // Data for response + transform(tc, slotName, ph) + + // Data for publish + c.Arc = fmt.Sprintf("%s", slotName) + c.Dst = ph } col = append(col, c) }(slot, tc) @@ -225,3 +233,13 @@ func transform(c *template.Collector, slot string, arc map[string]string) { } *c = p } + +func (t *tor) torrc() string { + if t.opts.TorrcFile() == "" { + return "" + } + if _, err := os.Open(t.opts.TorrcFile()); err != nil { + return "" + } + return t.opts.TorrcFile() +} diff --git a/service/telegram/telegram.go b/service/telegram/telegram.go index c50442ff..64171bbc 100644 --- a/service/telegram/telegram.go +++ b/service/telegram/telegram.go @@ -112,19 +112,20 @@ func (t *telegram) archive(urls []string) (col []*publish.Collect, err error) { go func(slot string) { defer wg.Done() c := &publish.Collect{} + logger.Debug("Telegram: archiving slot: %s", slot) switch slot { case config.SLOT_IA: - logger.Debug("Telegram: archiving slot: %s", slot) c.Arc = fmt.Sprintf("%s", config.SlotName(slot)) c.Dst = wbrc.IA() case config.SLOT_IS: - logger.Debug("Telegram: archiving slot: %s", slot) c.Arc = fmt.Sprintf("%s", config.SlotName(slot)) c.Dst = wbrc.IS() case config.SLOT_IP: - logger.Debug("Telegram: archiving slot: %s", slot) c.Arc = fmt.Sprintf("%s", config.SlotName(slot)) c.Dst = wbrc.IP() + case config.SLOT_PH: + c.Arc = fmt.Sprintf("%s", config.SlotName(slot)) + c.Dst = wbrc.PH() } col = append(col, c) }(slot) diff --git a/wayback.go b/wayback.go index f6a1a5d6..881477dd 100644 --- a/wayback.go +++ b/wayback.go @@ -7,6 +7,7 @@ package wayback // import "github.com/wabarc/wayback" import ( "github.com/wabarc/archive.is/pkg" "github.com/wabarc/archive.org/pkg" + "github.com/wabarc/telegra.ph/pkg" "github.com/wabarc/wayback/config" "github.com/wabarc/wayback/logger" "github.com/wabarc/wbipfs" @@ -21,6 +22,7 @@ type Broker interface { IA() Archived IS() Archived IP() Archived + PH() Archived } // Handle URLs need to wayback and configs, @@ -45,7 +47,7 @@ func (h *Handle) IS() Archived { wbrc := &is.Archiver{} uris, err := wbrc.Wayback(h.URLs) if err != nil { - logger.Error("Wayback %v to Archive.today failed, %v", h.URLs, err) + logger.Error("Wayback %v to archive.today failed, %v", h.URLs, err) } return uris @@ -65,3 +67,13 @@ func (h *Handle) IP() Archived { return uris } + +func (h *Handle) PH() Archived { + wbrc := &ph.Archiver{} + uris, err := wbrc.Wayback(h.URLs) + if err != nil { + logger.Error("Wayback %v to Telegra.ph failed, %v", h.URLs, err) + } + + return uris +}