diff --git a/.bump-version.el b/.bump-version.el index 99f3f2b..ca878e8 100644 --- a/.bump-version.el +++ b/.bump-version.el @@ -6,5 +6,7 @@ "google-translate-default-ui.el" "google-translate-pkg.el" "google-translate-smooth-ui.el" + "google-translate-cache.el" + "google-translate-cache-ui.el" "Makefile")) - (:current-version "0.12.0")) + (:current-version "0.12.1")) diff --git a/README.md b/README.md index 0acd096..1354316 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ displayed if available. If you want to see the phonetics, set this variable to t. The variable `google-translate-listen-program` determines the program -to use to listen translations. By default the program looks for +to use to listen to translations. By default the program looks for `mplayer` in the PATH, if `mplayer` is found then listening function will be available and you'll see `Listen` button in the buffer with the translation. You can use any other suitable program. If you use @@ -351,6 +351,34 @@ Additionally, these variables would be useful for troubleshooting: buffer `*google-translate-backend-debug*` (defaults to nil) +## Cache + +Translation results may be cached by setting `google-translate-use-cache` +to `t` (default). Only the text is cached, not the audio from `[Listen]`. + +Some customization variables: + +- `google-translate-cache-files-per-language` specifies how many files + may be used for storing the cache on the disk per language pair + (for incremental loading and saving), + +- `google-translate-cache-word-limit` specifies maximum words a request + (w/out the translation) may have in order to be cached. `nil` for no + limit. + +- `google-translate-cache-downcase-requests` indicates whether the text should + be downcased before translation (when it doesn't exceed + `google-translate-cache-word-limit`, that is) (defaults to `t`), + +- `google-translate-cache-directory` is the directory where the cache is to + be saved (defaults to `~/.emacs.d/var/translate-cache`). + +For batch word caching (such as vocabulary), see +`google-translate-cache-words-in-region` and +`google-translate-cache-words-in-buffer`. + +`google-translate-cache-save` is added to `kill-emacs-hook`. + ## Contributors - Tassilo Horn @@ -361,3 +389,4 @@ Additionally, these variables would be useful for troubleshooting: - [Michihito Shigemura](https://github.com/shigemk2) - [Tomotaka SUWA](https://github.com/t-suwa) - [stardiviner](https://github.com/stardiviner) +- Dmitrii Korobeinikov diff --git a/google-translate-cache-ui.el b/google-translate-cache-ui.el new file mode 100644 index 0000000..7012d5e --- /dev/null +++ b/google-translate-cache-ui.el @@ -0,0 +1,92 @@ +;;; google-translate-cache-ui.el --- Caching translation results UI + +;; Copyright (C) 2012 Oleksandr Manzyuk + +;; Author: Oleksandr Manzyuk +;; Maintainer: Andrey Tykhonov +;; URL: https://github.com/atykhonov/google-translate +;; Package-Requires: ((emacs "24.3") (popup "0.5.8")) +;; Version: 0.12.1 +;; Keywords: convenience + +;; Contributors: +;; Tassilo Horn +;; Bernard Hurley +;; Chris Bilson +;; Takumi Kinjo +;; momomo5717 +;; stardiviner +;; Dmitrii Korobeinikov + +;; This file is NOT part of GNU Emacs. + +;; This is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 2, or (at your option) +;; any later version. + +;; This file is distributed in the hope that it will be useful, but +;; WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with GNU Emacs. If not, see . + +;;; Commentary: + +;; UI for caching. +;; +;; `google-translate-cache-words-in-buffer' - retreives and caches +;; translations for every word in the buffer. +;; +;; `google-translate-cache-words-in-region' - retreives and caches +;; translations for every word in the active region. +;; + +;;; Code: + +(eval-when-compile (require 'cl-lib)) +(require 'google-translate-default-ui) + +(defun %google-translate-cache-words-in-region (beg end &optional override-p) + "Translate and cache words between BEG and END. + +For the meaning of OVERRIDE-P, see `google-translate-query-translate'." + (let ((langs (google-translate-read-args override-p nil)) + (word 1) + (total (count-words beg end))) + (save-excursion + (save-restriction + (narrow-to-region beg end) + (goto-char (point-min)) + (while (forward-word-strictly 1) + (message "Processing word %d/%d." word total) + (save-excursion + (google-translate-translate + (car langs) + (cadr langs) + (let ((b (bounds-of-thing-at-point 'word))) + (buffer-substring-no-properties (car b) (cdr b))))) + (cl-incf word)))))) + + +;;;###autoload +(defun google-translate-cache-words-in-region (beg end &optional override-p) + "Cache translations for all words in active region (from BEG to END). + +For the meaning of OVERRIDE-P, see `google-translate-query-translate'." + (interactive "rP") + (%google-translate-cache-words-in-region beg end override-p)) + +;;;###autoload +(defun google-translate-cache-words-in-buffer (&optional override-p) + "Cache translations for all words in current buffer. + +For the meaning of OVERRIDE-P, see `google-translate-query-translate'." + (interactive "P") + (%google-translate-cache-words-in-region (buffer-end -1) (buffer-end 1) override-p)) + +(provide 'google-translate-cache-ui) + +;;; google-translate-cache-ui.el ends here diff --git a/google-translate-cache.el b/google-translate-cache.el new file mode 100644 index 0000000..e3da164 --- /dev/null +++ b/google-translate-cache.el @@ -0,0 +1,255 @@ +;;; google-translate-cache.el --- Caching translation results + +;; Copyright (C) 2012 Oleksandr Manzyuk + +;; Author: Oleksandr Manzyuk +;; Maintainer: Andrey Tykhonov +;; URL: https://github.com/atykhonov/google-translate +;; Version: 0.12.1 +;; Keywords: convenience + +;; Contributors: +;; Tassilo Horn +;; Bernard Hurley +;; Chris Bilson +;; Takumi Kinjo +;; momomo5717 +;; stardiviner +;; Dmitrii Korobeinikov + +;; This file is NOT part of GNU Emacs. + +;; This is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 2, or (at your option) +;; any later version. + +;; This file is distributed in the hope that it will be useful, but +;; WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with GNU Emacs. If not, see . + +;;; Commentary: + +;; Caches translation results. + +;; Segmentation into multiple files is used, so that when the cache +;; is needed, there's no need to load everything. + +;;; Code: + +(eval-when-compile (require 'cl-lib)) +(require 'recentf) +(require 'dired) + + +(defvar google-translate-cache-files nil + "Holds all loaded and newly created cache-related files as a list of \ +elements with form (filename . plist). See \ +`google-translate--cache-get-file' for the format of the plist.") + + +(defgroup google-translate-cache nil + "Google Translate cache script." + :group 'processes) + + +(defcustom google-translate-use-cache t + "Whether the caching is on. + +Only strings containing less words than `google-translate-cache-word-limit' +are cached. These strings also get downcased beforehand if +`google-translate-downcase-cached-text' is non-nil." + :type '(boolean) + :group 'google-translate-cache) + +(defcustom google-translate-cache-directory "~/.emacs.d/var/translate-cache" + "Where the cache is kept." + :type '(string) + :group 'google-translate-cache) + +(defcustom google-translate-cache-downcase-requests t + "Set to non-nil to downcase translation requests. + +This helps to cache words which are in the beginning of a sentence." + :type '(boolean) + :group 'google-translate-cache) + +(defcustom google-translate-cache-files-per-language 101 + "How many files to keep cache in (per language pair). + +Cache may get quite large and keeping it all in many files has the advantage of +not having to load/save all of it at once, but rather in chunks. This variable +is how many chunks there will be (at most). Cache will be automatically rebuilt +once you change this variable, on first cache access." + :type '(integer) + :group 'google-translate-cache) + +(defcustom google-translate-cache-word-limit 2 + "Don't cache strings which have more words than this. +The number of words is deduced from `split-string'. +Set to nil for no limit." + :type '(integer) + :group 'google-translate-cache) + +(defmacro google-translate--cache-preserve-recentf (&rest body) + "Save a copy of `recentf-list' before executing BODY and then restore it. + +Because cache files don't need to be in recentf." + (let ((recentf-restore (gensym)) + (res (gensym))) + `(let ((,recentf-restore (copy-sequence recentf-list)) + (,res (progn ,@body))) + (setf recentf-list ,recentf-restore) + ,res))) + +(defun google-translate--cache-read-file (path) + "`Read' file located at PATH. Return nil if the file doesn't exist." + (google-translate--cache-preserve-recentf + (if (file-exists-p path) + (cadr (let* ((buf (find-file-noselect path)) + (contents (read buf))) + (kill-buffer buf) + contents))))) + +(defun google-translate--cache-expand-rpath (rpath) + "Expand RPATH as path relative to `google-translate-cache-directory'." + (expand-file-name + (concat google-translate-cache-directory + (unless (equal (substring google-translate-cache-directory -1) "/") "/") + rpath))) + +(defun google-translate--cache-get-file (rpath) + "Load file `google-translate-cache-directory'/RPATH. + +Return it as a property list, :dirty as nil and :contents as the contents of \ +the file. :dirty being non-nil indicates the need to be saved by \ +`google-translate-cache-save'. + +If the file doesn't exist, return the :contents as nil." + (let ((path (google-translate--cache-expand-rpath rpath))) + (cl-flet ((get () (alist-get path google-translate-cache-files nil nil 'equal))) + (or (get) + (progn + (push (cons path (list :dirty nil :contents (google-translate--cache-read-file path))) + google-translate-cache-files) + (get)))))) + +(defun google-translate--cache-rebuild-helper (source-language target-language cached-max) + "Rebuild cache when `google-translate-cache-files-per-language' doesn't match\ +CACHED-MAX found in SOURCE-LANGUAGE/TARGET-LANGUAGE/max-files." + (message "Rebuilding Google Translate cache...") + (let ((contents nil) + (x 0)) + ;; clean out the existing files collecting the cache into `contents' + (cl-loop for x from 0 to cached-max + do (let* ((name (concat source-language "/" target-language "/cache-" (int-to-string x))) + (segment (google-translate--cache-get-file name))) + (setf contents (append contents (plist-get segment :contents))) + (delete-file (google-translate--cache-expand-rpath name)) + (setf google-translate-cache-files + (assoc-delete-all (google-translate--cache-expand-rpath name) + google-translate-cache-files)))) + ;; rebuild cache from `contents' + (dolist (x contents) + (google-translate--cache-add-helper source-language target-language (car x) (cdr x))))) + +(defun google-translate--cache-rebuild-if-necessary (source-language target-language) + "Recache from SOURCE-LANGUAGE to TARGET-LANGUAGE to fit `google-translate-cache-files-per-language'." + (let* ((max-files (google-translate--cache-get-file + (concat source-language "/" target-language "/max-files"))) + (cached-max (plist-get max-files :contents)) + (needed-max google-translate-cache-files-per-language)) + (if (null cached-max) + (progn (plist-put max-files :contents needed-max) + (plist-put max-files :dirty t)) + (when (not (equal cached-max needed-max)) + (google-translate--cache-rebuild-helper source-language target-language cached-max) + (plist-put max-files :contents google-translate-cache-files-per-language) + (plist-put max-files :dirty t) + (google-translate-cache-save))))) + +(defun google-translate--cache-get-hash (key) + "Return cache file # where to keep string KEY." + ;; (mod (seq-reduce '+ key 0) google-translate-cache-files-per-language) + ;; testing on ~4000 cached English words, the following produces a more + ;; uniform distribution in comparison to the simple summation above + ;; (28--440kb -> 116--256kb per file across 101 files) + (cl-loop for c across key + for i from 0 to (length key) + with s = 0 + do (setf s (mod (+ s (* c (1+ (mod i 127)))) + google-translate-cache-files-per-language)) + finally return s)) + +(defun google-translate--cache-get-segment (source-language target-language key) + "Return file which should contain KEY (from SOURCE-LANGUAGE to TARGET-LANGUAGE)." + (google-translate--cache-get-file + (concat source-language + "/" + target-language + "/cache-" + (int-to-string (google-translate--cache-get-hash key))))) + +(defun google-translate--cache-add-helper (source-language target-language key translation) + "Cache TRANSLATION for KEY from SOURCE-LANGUAGE to TARGET-LANGUAGE. + +`google-translate--cache-rebuild-helper' is calling this functions instead of +`google-translate-cache-add', because that one calls +`google-translate--cache-rebuild-if-necessary'." + (let ((cache-segment + (google-translate--cache-get-segment source-language target-language key))) + ;; remove duplicates + (plist-put cache-segment :contents + (assoc-delete-all key (plist-get cache-segment :contents))) + ;; push (KEY . TRANSLATION) + (plist-put cache-segment :contents + (cons (cons key translation) + (plist-get cache-segment :contents))) + ;; mark segment as modified + (plist-put cache-segment :dirty t))) + +(defun google-translate-cache-add (source-language target-language key translation) + "Cache TRANSLATION for KEY from SOURCE-LANGUAGE to TARGET-LANGUAGE. + +`google-translate--key-to-segment-id' decides to which file the KEY goes to." + (google-translate--cache-rebuild-if-necessary source-language target-language) + (google-translate--cache-add-helper source-language target-language key translation) + translation) + +(defun google-translate-cache-get (source-language target-language key) + "Get cached translation for KEY from SOURCE-LANGUAGE to TARGET-LANGUAGE." + (google-translate--cache-rebuild-if-necessary source-language target-language) + (let ((cache-segment + (google-translate--cache-get-segment source-language target-language key))) + (alist-get key (plist-get cache-segment :contents) nil nil 'equal))) + +(defun google-translate-cache-save () + "Save cache to disk." + (google-translate--cache-preserve-recentf + (cl-loop for f in google-translate-cache-files + do (save-excursion + (if (plist-get (cdr f) :dirty) + (let ((buf (find-file-noselect (car f))) + (dir (file-name-directory (car f)))) + ;; Ensure the existence of the directory. + ;; Otherwise the user gets a dialogue about this + ;; (aka "you want to create this dir?") when emacs exits. + (unless (f-exists-p dir) + (dired-create-directory dir)) + (plist-put (cdr f) :dirty nil) + (set-buffer buf) + (erase-buffer) + (insert "'") + (insert (prin1-to-string (plist-get (cdr f) :contents))) + (save-buffer) + (kill-buffer))))))) + +(add-hook 'kill-emacs-hook 'google-translate-cache-save) + +(provide 'google-translate-cache) + +;;; google-translate-cache.el ends here diff --git a/google-translate-core-ui.el b/google-translate-core-ui.el index 2bc4af5..c0d1779 100644 --- a/google-translate-core-ui.el +++ b/google-translate-core-ui.el @@ -171,7 +171,6 @@ (eval-when-compile (require 'cl-lib)) (require 'google-translate-core) (require 'ido) -(require 'popup) (require 'color) (defvar google-translate-supported-languages-alist @@ -288,7 +287,7 @@ query parameter in HTTP requests.") (defvar google-translate-translation-listening-debug nil "For debug translation listening purposes.") -(defstruct gtos +(cl-defstruct gtos "google translate output structure contains miscellaneous information which intended to be outputed to the buffer, echo area or popup tooltip." diff --git a/google-translate-core.el b/google-translate-core.el index f03be23..ee4375c 100644 --- a/google-translate-core.el +++ b/google-translate-core.el @@ -6,7 +6,7 @@ ;; Maintainer: Andrey Tykhonov ;; URL: https://github.com/atykhonov/google-translate ;; Package-Requires: ((emacs "25.1")) -;; Version: 0.12.0 +;; Version: 0.12.1 ;; Keywords: convenience ;; Contributors: @@ -16,6 +16,7 @@ ;; Takumi Kinjo ;; momomo5717 ;; stardiviner +;; Dmitrii Korobeinikov ;; This file is NOT part of GNU Emacs. @@ -70,6 +71,7 @@ (require 'json) (require 'url) (require 'google-translate-tk) +(require 'google-translate-cache) (defgroup google-translate-core nil "Google Translate core script." @@ -238,12 +240,32 @@ request." translate TEXT from SOURCE-LANGUAGE to TARGET-LANGUAGE. Returns response in json format." (let ((cleaned-text (google-translate-prepare-text-for-request text))) - (when (and - (stringp cleaned-text) - (> (length cleaned-text) 0)) - (json-read-from-string - (google-translate--insert-nulls - (google-translate--request source-language target-language text)))))) + (when (and (stringp cleaned-text) + (> (length cleaned-text) 0)) + (cl-flet ((translate () (json-read-from-string + (google-translate--insert-nulls + (google-translate--request + source-language + target-language + text))))) + (if (and google-translate-use-cache + (or (null google-translate-cache-word-limit) + (<= (length (split-string cleaned-text)) + google-translate-cache-word-limit))) + (progn + (if google-translate-cache-downcase-requests + ;; rely on dynamic scoping + (progn + (setq cleaned-text (downcase cleaned-text)) + (setq text (downcase text)))) + (or (google-translate-cache-get source-language + target-language + cleaned-text) + (google-translate-cache-add source-language + target-language + cleaned-text + (translate)))) + (translate)))))) (defun google-translate--request (source-language target-language @@ -333,7 +355,7 @@ translation it is possible to get suggestion." (defun google-translate-version () (interactive) - (message "Google Translate (version): %s" "0.12.0")) + (message "Google Translate (version): %s" "0.12.1")) (provide 'google-translate-core) diff --git a/google-translate-pkg.el b/google-translate-pkg.el index 6418fec..f34c8e1 100644 --- a/google-translate-pkg.el +++ b/google-translate-pkg.el @@ -1,2 +1,2 @@ -(define-package "google-translate" "0.12.0" +(define-package "google-translate" "0.12.1" "Emacs interface to Google Translate.") diff --git a/test/google-translate-cache-test.el b/test/google-translate-cache-test.el new file mode 100644 index 0000000..28c9386 --- /dev/null +++ b/test/google-translate-cache-test.el @@ -0,0 +1,48 @@ +(require 'dired) + +(defmacro with-cache-context (&rest body) + "Set up some variable, run BODY and clean up." + `(let* ((google-translate-use-cache t) + (source "tmp-test-source") + (target "tmp-test-target") + (prefix (google-translate--cache-expand-rpath (concat source "/" target "/")))) + ;; When a directory doesn't already exist, emacs asks user for permission to create it. + ;; That won't work as part of a test. + ;; So have to ensure the existence manually here. + (unwind-protect + (progn (unless (f-exists-p prefix) + (dired-create-directory prefix)) + ,@body) + (progn + (delete-directory (google-translate--cache-expand-rpath source) t) + (setf google-translate-cache-files nil))))) + +(ert-deftest test-google-translate-cache-save-and-load () + "Test cache addition/retrieval + file saving/loading." + (with-cache-context + (let* ((key "test-word") + (translation "test-translation") + (cache-file-path (concat prefix "cache-" (int-to-string (google-translate--cache-get-hash key))))) + ;; add + (google-translate-cache-add source target key translation) + ;; save to file + (google-translate-cache-save) + (should (file-exists-p cache-file-path)) + (should (file-exists-p (concat prefix "max-files"))) + (setf google-translate-cache-files nil) + ;; load and retrieve + (should (equal (google-translate-cache-get source target key) + translation))))) + +(ert-deftest test-google-translate-cache-rebuild () + "Test cache rebuilding (when `google-translate-cache-files-per-language' changes)." + (with-cache-context + (let ((google-translate-cache-files-per-language 100) + (pairs '(("a" "b") ("c" "d") ("e" "f") ("rnsotiearst" "rsteer")))) + (dolist (x pairs) + (google-translate-cache-add source target (car x) (cadr x))) + (google-translate-cache-save) + (setf google-translate-cache-files-per-language 5) + (dolist (x pairs) + (should (equal (google-translate-cache-get source target (car x)) + (cadr x)))))))