feat: first draft version

This commit is contained in:
Pavel Korytov 2022-05-26 22:11:27 +03:00
parent d607158012
commit 65ca6e1564
2 changed files with 434 additions and 5 deletions

View file

@ -28,6 +28,8 @@
;;; Code: ;;; Code:
(require 'elfeed) (require 'elfeed)
(require 'seq)
(require 'elfeed-db)
(require 'request) (require 'request)
(defgroup elfeed-sync () (defgroup elfeed-sync ()
@ -49,9 +51,104 @@
:group 'elfeed-sync :group 'elfeed-sync
:type 'string) :type 'string)
(defcustom elfeed-sync-look-back 15552000
"How far back to sync."
:group 'elfeed-sync
:type 'number)
(defcustom elfeed-sync-unread-tag 'unread
"Unread tag for sync."
:group 'elfeed-sync
:type 'symbol)
(defcustom elfeed-sync-marked-tag 'later
"Marked tag for sync."
:group 'elfeed-sync
:type 'symbol)
(defcustom elfeed-sync-missing-attempts 5
"How many attempts to sync missing entries."
:group 'elfeed-sync
:type 'number)
(defcustom elfeed-sync-missing-targets-keep (* 60 60 24 7)
"How long to keep missing targets."
:group 'elfeed-sync
:type 'number)
(defvar elfeed-sync--tt-rss-sid nil (defvar elfeed-sync--tt-rss-sid nil
"Session ID.") "Session ID.")
(defvar elfeed-sync--state nil
"State of the tt-rss sync.")
(cl-defstruct (elfeed-sync-datum (:constructor elfeed-sync-datum--create))
id tags)
(defun elfeed-sync--state-empty ()
"Create an empty elfeed-sync state."
`((:last-sync . nil)
(:feeds . ,(make-hash-table :test #'equal))
(:missing . ,(make-hash-table :test #'equal))
(:discarded . ,(make-hash-table :test #'equal))
(:missing-target . nil)))
(defun elfeed-sync--state-file ()
(concat elfeed-db-directory "/sync-state"))
(defun elfeed-sync--state-load ()
"Load elfeed-sync state from the filesystem."
(if (not (file-exists-p (elfeed-sync--state-file)))
(setf elfeed-sync--state (elfeed-sync--state-empty))
(with-temp-buffer
(insert-file-contents (elfeed-sync--state-file))
(goto-char (point-min))
(condition-case _
(progn
(setf elfeed-sync--state (read (current-buffer))))
(error (progn
(message "Recreating the sync state because of read error")
(setf elfeed-sync--state (elfeed-sync--state-empty))))))))
(defun elfeed-sync--state-ensure ()
"Ensure that the sync state has been loaded."
(when (null elfeed-sync--state) (elfeed-sync--state-load)))
(defun elfeed-sync-reset ()
"Reset the elfeed sync state"
(interactive)
(setf elfeed-sync--state (elfeed-sync--state-empty)))
(defun elfeed-sync-state-save ()
"Save the elfeed sync state to the filesystem."
(interactive)
(elfeed-sync--state-ensure)
(mkdir (file-name-directory (elfeed-sync--state-file)) t)
(let ((coding-system-for-write 'utf-8))
(with-temp-file (elfeed-sync--state-file)
(let ((standard-output (current-buffer))
(print-level nil)
(print-length nil)
(print-circle nil))
(princ ";;; Elfeed Sync State\n\n")
(prin1 elfeed-sync--state)))))
(defun elfeed-sync-state-save-safe ()
"Save the elfeed sync state, ignoring errors.
This can be put to `kill-emacs-hook' and not screw up anything
with exceptions."
(ignore-errors
(org-journal-tags-db-save)))
(defun elfeed-sync--json-read-safe ()
(condition-case _
(json-read)
(with-current-buffer (get-buffer-create "*elfeed-sync-json-read-error*")
(erase-buffer)
(insert-buffer-substring (current-buffer))
(error "Error reading JSON: %s" (buffer-string)))))
(defun elfeed-sync--with-session (callback) (defun elfeed-sync--with-session (callback)
"Run CALLBACK with the active tt-rss session. "Run CALLBACK with the active tt-rss session.
@ -67,7 +164,7 @@ called only after a succesful login query."
("user" . ,elfeed-sync-tt-rss-login) ("user" . ,elfeed-sync-tt-rss-login)
("password" . ,elfeed-sync-tt-rss-password))) ("password" . ,elfeed-sync-tt-rss-password)))
:headers '(("Content-Type" . "application/json")) :headers '(("Content-Type" . "application/json"))
:parser 'json-read :parser 'elfeed-sync--json-read-safe
:success (elfeed-sync--handler :success (elfeed-sync--handler
(setq elfeed-sync--tt-rss-sid (setq elfeed-sync--tt-rss-sid
(alist-get 'session_id (alist-get 'session_id
@ -144,12 +241,226 @@ called only after a succesful login query."
("sid" . ,elfeed-sync--tt-rss-sid) ("sid" . ,elfeed-sync--tt-rss-sid)
("tree" . ,(elfeed-sync--get-tree)))) ("tree" . ,(elfeed-sync--get-tree))))
:headers '(("Content-Type" . "application/json")) :headers '(("Content-Type" . "application/json"))
:parser 'json-read :parser 'elfeed-sync--json-read-safe
:success (elfeed-sync--handler :success (elfeed-sync--handler
(message "Success!")) (message "Success!"))
:error :error
(cl-function (lambda (&key error-thrown &allow-other-keys) (cl-function (lambda (&key error-thrown &allow-other-keys)
(message "Error: %S" error-thrown)))))) (message "Error: %S" error-thrown))))))
(defun elfeed-sync--get-bad-feeds ()
(let ((feed-hash (make-hash-table :test #'equal))
(bad-feeds-hash (make-hash-table :test #'equal)))
(with-elfeed-db-visit (entry feed)
(let ((url-hash (gethash (elfeed-feed-id feed) feed-hash)))
(unless url-hash
(setq url-hash (make-hash-table :test #'equal))
(puthash (elfeed-feed-id feed) url-hash feed-hash))
(if (gethash (elfeed-entry-link entry) url-hash)
(puthash (elfeed-feed-id feed) t bad-feeds-hash)
(puthash (elfeed-entry-link entry) t url-hash)))
(when (> (- (time-convert nil 'integer)
elfeed-sync-look-back)
(elfeed-entry-date entry))
(elfeed-db-return)))
bad-feeds-hash))
(defun elfeed-sync--datum-chaged (datum entry)
(let* ((old-tags (elfeed-sync-datum-tags datum))
(new-tags (elfeed-entry-tags entry))
(common-tags (seq-intersection old-tags new-tags)))
(or (not (= (length old-tags) (length common-tags)))
(not (= (length new-tags) (length common-tags))))))
(defun elfeed-sync--entry-good-p (id)
(null (gethash id (alist-get :discarded elfeed-sync--state))))
(defun elfeed-sync--get-changed ()
(let ((feed-hash (alist-get :feeds elfeed-sync--state))
changed)
(with-elfeed-db-visit (entry feed)
(when-let ((id (elfeed-ref-id (elfeed-entry-content entry))))
(when (elfeed-sync--entry-good-p id)
(let ((entry-hash (gethash (elfeed-feed-id feed) feed-hash)))
(unless entry-hash
(setq entry-hash (make-hash-table :test #'equal))
(puthash (elfeed-feed-id feed) entry-hash feed-hash))
(if-let ((datum (gethash id entry-hash)))
(when (elfeed-sync--datum-chaged datum entry)
(push (list entry feed id) changed))
(push (list entry feed id) changed)))))
(when (> (- (time-convert nil 'integer)
elfeed-sync-look-back)
(elfeed-entry-date entry))
(elfeed-db-return)))
changed))
(defun elfeed-sync--prepare-request (bad-feeds)
(let ((changed (elfeed-sync--get-changed)))
`((bad_feeds . ,(cl-loop for key being the hash-keys of bad-feeds
collect key))
(changed .,(mapcar
(lambda (datum)
(let ((entry (nth 0 datum))
(feed (nth 1 datum))
(id (nth 2 datum)))
(if (gethash (or (elfeed-feed-url feed)
(elfeed-feed-id feed))
bad-feeds)
`((id . ,id)
(title . ,(elfeed-entry-title entry))
(url . ,(elfeed-entry-link entry))
(feed_url . ,(or (elfeed-feed-url feed)
(elfeed-feed-id feed)))
(updated . ,(format-time-string
"%Y-%m-%d %H:%M:%S"
(seconds-to-time (elfeed-entry-date entry))
"UTC0"))
(tags . ,(elfeed-entry-tags entry)))
`((id . ,id)
(feed_url . ,(or (elfeed-feed-url feed)
(elfeed-feed-id feed)))
(url . ,(elfeed-entry-link entry))
(tags . ,(elfeed-entry-tags entry))))))
changed))
(last_sync . ,(alist-get :last-sync elfeed-sync--state))
(unread_tag . ,elfeed-sync-unread-tag)
(marked_tag . ,elfeed-sync-marked-tag)
(look_back . ,elfeed-sync-look-back))))
(defun elfeed-sync--sort-response-entries (entries bad-feeds entries-by-title-date entries-by-url is-new)
(dolist (entry entries)
(setf (alist-get 'is-new entry) is-new)
(if (gethash (alist-get 'feed_url entry) bad-feeds)
(let ((title-date (format "%s---%s"
(alist-get 'title entry)
(alist-get 'updated entry))))
(puthash title-date entry entries-by-title-date))
(puthash (alist-get 'link entry) entry entries-by-url))))
(defun elfeed-sync--get-response-entry (entry feed entries-by-title-date entries-by-url)
(if (gethash (or (elfeed-feed-url feed)
(elfeed-feed-id feed))
bad-feeds)
(let ((title-date (format "%s---%s"
(elfeed-entry-title entry)
(format-time-string
"%Y-%m-%d %H:%M:%S"
(seconds-to-time (elfeed-entry-date entry))
"UTC0"))))
(prog1
(gethash title-date entries-by-title-date)
(remhash title-date entries-by-title-date)))
(prog1
(gethash (elfeed-entry-link entry) entries-by-url)
(remhash (elfeed-entry-link entry) entries-by-url))))
(defun elfeed-sync--set-entry-unread (entry status)
"Set the unread status of ENTRY to STATUS.
STATUS is a boolean. If nil, the entry is marked as read. ENTRY is an instance of `elfeed-entry'."
(let ((is-unread (member elfeed-sync-unread-tag
(elfeed-entry-tags entry))))
(if (and is-unread status)
(elfeed-untag entry elfeed-sync-unread-tag)
(when (not is-unread)
(elfeed-tag entry elfeed-sync-unread-tag)))))
(defun elfeed-sync--set-entry-marked (entry status)
"Set the marked status of ENTRY to STATUS.
STATUS is a boolean. If nil, the entry is marked as
unmarked. ENTRY is an instance of `elfeed-entry'."
(let ((is-marked (member elfeed-sync-marked-tag
(elfeed-entry-tags entry))))
(if (and is-marked status)
(elfeed-untag entry elfeed-sync-marked-tag)
(when (not is-marked)
(elfeed-tag entry elfeed-sync-marked-tag)))))
(defun elfeed-sync--process-response (response bad-feeds)
(cl-loop for entry in (alist-get 'missing-entries response)
do (let* ((id (alist-get 'id entry))
(attempts (or
(gethash id (alist-get :missing elfeed-sync--state))
0)))
(if (>= attempts elfeed-sync-max-retries)
(puthash id (1+ attempts) (alist-get :missing elfeed-sync--state))
(puthash id t (alist-get :discarded elfeed-sync--state))
(remhash id (alist-get :missing elfeed-sync--state)))))
(setf (alist-get :missing-target elfeed-sync--state)
(seq-filter (lambda (datum)
(< (- (time-convert nil 'integer) (car datum))
elfeed-sync-missing-targets-keep))
(alist-get :missing-target elfeed-sync--state)))
(let ((entries-by-title-date (make-hash-table :test #'equal))
(entries-by-url (make-hash-table :test #'equal)))
(elfeed-sync--sort-response-entries
(alist-get 'updated response) bad-feeds entries-by-title-date entries-by-url t)
(elfeed-sync--sort-response-entries
(mapcar #'cdr (alist-get :missing-target elfeed-sync--state))
bad-feeds entries-by-title-date entries-by-url nil)
(with-elfeed-db-visit (entry feed)
(when-let ((id (elfeed-ref-id (elfeed-entry-content entry))))
(if-let ((entry (elfeed-sync--get-response-entry entry feed entries-by-title-date entries-by-url)))
(progn
(elfeed-sync--set-entry-unread entry (alist-get 'unread entry))
(elfeed-sync--set-entry-marked entry (alist-get 'marked entry))
(puthash id (elfeed-sync-datum--create
:id id
:tags (elfeed-entry-tags entry))))
(unless (or (gethash id (alist-get :missing elfeed-sync--state))
(gethash id (alist-get :discarded elfeed-sync--state)))
(puthash id (elfeed-sync-datum--create
:id id
:tags (elfeed-entry-tags entry))
(gethash
(or (elfeed-feed-url feed)
(elfeed-feed-id feed))
(alist-get :feeds elfeed-sync--state))))))
(when (> (- (time-convert nil 'integer)
elfeed-sync-look-back)
(elfeed-entry-date entry))
(elfeed-db-return)))
(maphash (lambda (_ datum)
(when (alist-get 'is-new datum)
(push datum (alist-get :missing-target elfeed-sync--state))))
entries-by-title-date)
(maphash (lambda (_ datum)
(when (alist-get 'is-new datum)
(push datum (alist-get :missing-target elfeed-sync--state))))
entries-by-url))
(setf (alist-get :last-sync elfeed-sync--state)
(time-convert nil 'integer)))
(defun elfeed-sync ()
(interactive)
(elfeed-sync--session
(let* ((bad-feeds (elfeed-sync--get-bad-feeds))
(request (elfeed-sync--prepare-request bad-feeds)))
(request (concat elfeed-sync-tt-rss-instance "/api/")
:type "POST"
:data (json-encode
`(("op" . "syncElfeed")
("sid" . ,elfeed-sync--tt-rss-sid)
("data" . ,request)))
:parser 'elfeed-sync--json-read-safe
:headers '(("Content-Type" . "application/json"))
:success (elfeed-sync--handler
(elfeed-sync--process-response
(alist-get 'content data)
bad-feeds))))))
;;;###autoload
(define-minor-mode elfeed-sync-mode
"TODO"
:global t
(if elfeed-sync-mode
(progn
(add-hook 'kill-emacs-hook #'elfeed-sync-state-save-safe))
(remove-hook 'kill-emacs-hook #'elfeed-sync-state-save-safe)))
(provide 'elfeed-sync) (provide 'elfeed-sync)
;;; elfeed-sync.el ends here ;;; elfeed-sync.el ends here

124
init.php
View file

@ -14,6 +14,7 @@ class Elfeed_Sync extends Plugin {
$this->host = $host; $this->host = $host;
$this->host->add_api_method("setFeedsTree", $this); $this->host->add_api_method("setFeedsTree", $this);
$this->host->add_api_method("syncElfeed", $this);
} }
function createCategoriesTree($tree, $feed_categories = null, $current_cat = null) { function createCategoriesTree($tree, $feed_categories = null, $current_cat = null) {
@ -113,9 +114,6 @@ class Elfeed_Sync extends Plugin {
return $results; return $results;
} }
/**
* Our own API.
*/
function setFeedsTree() { function setFeedsTree() {
$tree = $_REQUEST["tree"]; $tree = $_REQUEST["tree"];
@ -124,4 +122,124 @@ class Elfeed_Sync extends Plugin {
return array(API::STATUS_OK, array('subscribed' => $subscribed)); return array(API::STATUS_OK, array('subscribed' => $subscribed));
} }
function updateEntries($data) {
$entries_query = ORM::for_table('ttrss_entries')
->table_alias('e')
->select_many('e.id', 'e.link', 'e.title', 'e.updated',
'ue.marked', 'ue.unread', 'ue.feed_id')
->join('ttrss_user_entries', ['ue.ref_id', '=', 'e.id'], 'ue')
->join('ttrss_feeds', ['f.id', '=', 'ue.feed_id'], 'f')
->where('ue.owner_uid', $_SESSION['uid']);
if (!is_null($data['look_back'])) {
$entries_query = $entries_query->where_gte('updated', date('Y-m-d H:i:s', time() - $data['look_back']));
}
$entries = $entries_query->find_array();
$feeds = ORM::for_table('ttrss_feeds')
->select_many('id', 'feed_url')
->where('owner_uid', $_SESSION['uid'])
->find_array();
$feed_by_link = array();
foreach ($feeds as $feed) {
$feed_by_link[$feed['feed_url']] = $feed['id'];
}
$bad_feed_ids = array();
foreach($data['bad_feeds'] as $bad_feed_link) {
$feed_id = $feed_by_link[$bad_feed_link];
if ($feed_id) {
$bad_feed_ids[$feed_id] = true;
}
}
$entries_by_link = array();
$entries_by_title_date = array();
foreach ($entries as $entry) {
if ($bad_feed_ids[$entry['feed_id']]) {
$title_date = $entry['title'] . '---' . $entry['updated'];
$entries_by_title_date[$title_date] = $entry;
} else {
$entries_by_link[$entry['link']] = $entry;
}
}
$missing_feeds = array();
$missing_entries = array();
$toggle_unread = array();
$toggle_marked = array();
foreach($data['changed'] as $changed_entry) {
$feed_id = $feed_by_link[$changed_entry['feed_url']];
if (!$feed_id) {
$missing_feeds[$changed_entry['feed_url']] = true;
continue;
}
$bad_feed = $bad_feed_ids[$feed_id];
$entry = null;
if ($bad_feed) {
$title_date = $changed_entry['title'] . '---' . $changed_entry['date'];
$entry = $entries_by_title_date[$title_date];
} else {
$entry = $entries_by_link[$changed_entry['url']];
}
if (!$entry) {
array_push($missing_entries, $changed_entry);
continue;
}
if (!is_array($changed_entry['tags'])) {
continue;
}
$unread = in_array($data['unread_tag'], $changed_entry['tags']);
$marked = in_array($data['starred_tag'], $changed_entry['tags']);
if ($entry['unread'] != $unread) {
array_push($toggle_unread, $entry['id']);
}
if ($entry['marked'] != $marked) {
array_push($toggle_marked, $entry['id']);
}
}
ORM::get_db()->beginTransaction();
if (count($toggle_unread) > 0) {
ORM::raw_execute('UPDATE ttrss_user_entries t SET unread = not t.unread WHERE ref_id IN ('.implode(',', $toggle_unread).')');
}
if (count($toggle_marked) > 0) {
ORM::raw_execute('UPDATE ttrss_user_entries t SET marked = not t.marked WHERE ref_id IN ('.implode(',', $toggle_marked).')');
}
ORM::get_db()->commit();
return array($missing_feeds, $missing_entries, sizeof($toggle_unread), sizeof($toggle_marked));
}
function getUpdatedEntries($data) {
if (is_null($data['last_sync'])) {
return [];
}
$last_sync = date('Y-m-d H:i:s', $data['last_sync']);
$entries = ORM::for_table('ttrss_entries')
->select_many('link', 'ttrss_entries.title', 'updated', 'feed_url', 'marked', 'unread')
->join('ttrss_user_entries', ['ttrss_entries.id', '=', 'ttrss_user_entries.ref_id'])
->join('ttrss_feeds', ['ttrss_feeds.id', '=', 'ttrss_user_entries.feed_id'])
->where_raw('ttrss_user_entries.last_read >= ? OR ttrss_user_entries.last_marked >= ?', [$last_sync, $last_sync])
->where('ttrss_user_entries.owner_uid', $_SESSION['uid'])
->find_array();
return $entries;
}
function syncElfeed() {
$data = $_REQUEST["data"];
list($missing_feeds, $missing_entries, $toggled_unread, $toggled_marked) = $this->updateEntries($data);
$changed_entries = $this->getUpdatedEntries($data);
return array(API::STATUS_OK,
array(
'missing_feeds' => $missing_feeds,
'missing_entries' => $missing_entries,
'toggled_unread' => $toggled_unread,
'toggled_marked' => $toggled_marked,
'updated' => $changed_entries
)
);
}
} }