From 65ca6e15645cd0fb8f29e06b24b845a11bd94844 Mon Sep 17 00:00:00 2001
From: SqrtMinusOne <thexcloud@gmail.com>
Date: Thu, 26 May 2022 22:11:27 +0300
Subject: [PATCH] feat: first draft version

---
 elfeed-sync.el | 315 ++++++++++++++++++++++++++++++++++++++++++++++++-
 init.php       | 124 ++++++++++++++++++-
 2 files changed, 434 insertions(+), 5 deletions(-)

diff --git a/elfeed-sync.el b/elfeed-sync.el
index b2a34b8..012b190 100644
--- a/elfeed-sync.el
+++ b/elfeed-sync.el
@@ -28,6 +28,8 @@
 
 ;;; Code:
 (require 'elfeed)
+(require 'seq)
+(require 'elfeed-db)
 (require 'request)
 
 (defgroup elfeed-sync ()
@@ -49,9 +51,104 @@
   :group 'elfeed-sync
   :type 'string)
 
+(defcustom elfeed-sync-look-back 15552000
+  "How far back to sync."
+  :group 'elfeed-sync
+  :type 'number)
+
+(defcustom elfeed-sync-unread-tag 'unread
+  "Unread tag for sync."
+  :group 'elfeed-sync
+  :type 'symbol)
+
+(defcustom elfeed-sync-marked-tag 'later
+  "Marked tag for sync."
+  :group 'elfeed-sync
+  :type 'symbol)
+
+(defcustom elfeed-sync-missing-attempts 5
+  "How many attempts to sync missing entries."
+  :group 'elfeed-sync
+  :type 'number)
+
+(defcustom elfeed-sync-missing-targets-keep (* 60 60 24 7)
+  "How long to keep missing targets."
+  :group 'elfeed-sync
+  :type 'number)
+
 (defvar elfeed-sync--tt-rss-sid nil
   "Session ID.")
 
+(defvar elfeed-sync--state nil
+  "State of the tt-rss sync.")
+
+(cl-defstruct (elfeed-sync-datum (:constructor elfeed-sync-datum--create))
+  id tags)
+
+(defun elfeed-sync--state-empty ()
+  "Create an empty elfeed-sync state."
+  `((:last-sync . nil)
+    (:feeds . ,(make-hash-table :test #'equal))
+    (:missing . ,(make-hash-table :test #'equal))
+    (:discarded . ,(make-hash-table :test #'equal))
+    (:missing-target . nil)))
+
+(defun elfeed-sync--state-file ()
+  (concat elfeed-db-directory "/sync-state"))
+
+(defun elfeed-sync--state-load ()
+  "Load elfeed-sync state from the filesystem."
+  (if (not (file-exists-p (elfeed-sync--state-file)))
+      (setf elfeed-sync--state (elfeed-sync--state-empty))
+    (with-temp-buffer
+      (insert-file-contents (elfeed-sync--state-file))
+      (goto-char (point-min))
+      (condition-case _
+          (progn
+            (setf elfeed-sync--state (read (current-buffer))))
+        (error (progn
+                 (message "Recreating the sync state because of read error")
+                 (setf elfeed-sync--state (elfeed-sync--state-empty))))))))
+
+(defun elfeed-sync--state-ensure ()
+  "Ensure that the sync state has been loaded."
+  (when (null elfeed-sync--state) (elfeed-sync--state-load)))
+
+(defun elfeed-sync-reset ()
+  "Reset the elfeed sync state"
+  (interactive)
+  (setf elfeed-sync--state (elfeed-sync--state-empty)))
+
+(defun elfeed-sync-state-save ()
+  "Save the elfeed sync state to the filesystem."
+  (interactive)
+  (elfeed-sync--state-ensure)
+  (mkdir (file-name-directory (elfeed-sync--state-file)) t)
+  (let ((coding-system-for-write 'utf-8))
+    (with-temp-file (elfeed-sync--state-file)
+      (let ((standard-output (current-buffer))
+            (print-level nil)
+            (print-length nil)
+            (print-circle nil))
+        (princ ";;; Elfeed Sync State\n\n")
+        (prin1 elfeed-sync--state)))))
+
+(defun elfeed-sync-state-save-safe ()
+  "Save the elfeed sync state, ignoring errors.
+
+This can be put to `kill-emacs-hook' and not screw up anything
+with exceptions."
+  (ignore-errors
+    (org-journal-tags-db-save)))
+
+(defun elfeed-sync--json-read-safe ()
+  (condition-case _
+      (json-read)
+    (with-current-buffer (get-buffer-create "*elfeed-sync-json-read-error*")
+      (erase-buffer)
+      (insert-buffer-substring (current-buffer))
+      (error "Error reading JSON: %s" (buffer-string)))))
+
 (defun elfeed-sync--with-session (callback)
   "Run CALLBACK with the active tt-rss session.
 
@@ -67,7 +164,7 @@ called only after a succesful login query."
                ("user" . ,elfeed-sync-tt-rss-login)
                ("password" . ,elfeed-sync-tt-rss-password)))
       :headers '(("Content-Type" . "application/json"))
-      :parser 'json-read
+      :parser 'elfeed-sync--json-read-safe
       :success (elfeed-sync--handler
                 (setq elfeed-sync--tt-rss-sid
                       (alist-get 'session_id
@@ -144,12 +241,226 @@ called only after a succesful login query."
               ("sid" . ,elfeed-sync--tt-rss-sid)
               ("tree" . ,(elfeed-sync--get-tree))))
      :headers '(("Content-Type" . "application/json"))
-     :parser 'json-read
+     :parser 'elfeed-sync--json-read-safe
      :success (elfeed-sync--handler
                (message "Success!"))
      :error
      (cl-function (lambda (&key error-thrown &allow-other-keys)
                     (message "Error: %S" error-thrown))))))
 
+(defun elfeed-sync--get-bad-feeds ()
+  (let ((feed-hash (make-hash-table :test #'equal))
+        (bad-feeds-hash (make-hash-table :test #'equal)))
+    (with-elfeed-db-visit (entry feed)
+      (let ((url-hash (gethash (elfeed-feed-id feed) feed-hash)))
+        (unless url-hash
+          (setq url-hash (make-hash-table :test #'equal))
+          (puthash (elfeed-feed-id feed) url-hash feed-hash))
+        (if (gethash (elfeed-entry-link entry) url-hash)
+            (puthash (elfeed-feed-id feed) t bad-feeds-hash)
+          (puthash (elfeed-entry-link entry) t url-hash)))
+      (when (> (- (time-convert nil 'integer)
+                  elfeed-sync-look-back)
+               (elfeed-entry-date entry))
+        (elfeed-db-return)))
+    bad-feeds-hash))
+
+
+(defun elfeed-sync--datum-chaged (datum entry)
+  (let* ((old-tags (elfeed-sync-datum-tags datum))
+         (new-tags (elfeed-entry-tags entry))
+         (common-tags (seq-intersection old-tags new-tags)))
+    (or (not (= (length old-tags) (length common-tags)))
+        (not (= (length new-tags) (length common-tags))))))
+
+(defun elfeed-sync--entry-good-p (id)
+  (null (gethash id (alist-get :discarded elfeed-sync--state))))
+
+(defun elfeed-sync--get-changed ()
+  (let ((feed-hash (alist-get :feeds elfeed-sync--state))
+        changed)
+    (with-elfeed-db-visit (entry feed)
+      (when-let ((id (elfeed-ref-id (elfeed-entry-content entry))))
+        (when (elfeed-sync--entry-good-p id)
+          (let ((entry-hash (gethash (elfeed-feed-id feed) feed-hash)))
+            (unless entry-hash
+              (setq entry-hash (make-hash-table :test #'equal))
+              (puthash (elfeed-feed-id feed) entry-hash feed-hash))
+            (if-let ((datum (gethash id entry-hash)))
+                (when (elfeed-sync--datum-chaged datum entry)
+                  (push (list entry feed id) changed))
+              (push (list entry feed id) changed)))))
+      (when (> (- (time-convert nil 'integer)
+                  elfeed-sync-look-back)
+               (elfeed-entry-date entry))
+        (elfeed-db-return)))
+    changed))
+
+(defun elfeed-sync--prepare-request (bad-feeds)
+  (let ((changed (elfeed-sync--get-changed)))
+    `((bad_feeds . ,(cl-loop for key being the hash-keys of bad-feeds
+                             collect key))
+      (changed .,(mapcar
+                  (lambda (datum)
+                    (let ((entry (nth 0 datum))
+                          (feed (nth 1 datum))
+                          (id (nth 2 datum)))
+                      (if (gethash (or (elfeed-feed-url feed)
+                                       (elfeed-feed-id feed))
+                                   bad-feeds)
+                          `((id . ,id)
+                            (title . ,(elfeed-entry-title entry))
+                            (url . ,(elfeed-entry-link entry))
+                            (feed_url . ,(or (elfeed-feed-url feed)
+                                             (elfeed-feed-id feed)))
+                            (updated . ,(format-time-string
+                                         "%Y-%m-%d %H:%M:%S"
+                                         (seconds-to-time (elfeed-entry-date entry))
+                                         "UTC0"))
+                            (tags . ,(elfeed-entry-tags entry)))
+                        `((id . ,id)
+                          (feed_url . ,(or (elfeed-feed-url feed)
+                                           (elfeed-feed-id feed)))
+                          (url . ,(elfeed-entry-link entry))
+                          (tags . ,(elfeed-entry-tags entry))))))
+                  changed))
+      (last_sync . ,(alist-get :last-sync elfeed-sync--state))
+      (unread_tag . ,elfeed-sync-unread-tag)
+      (marked_tag . ,elfeed-sync-marked-tag)
+      (look_back . ,elfeed-sync-look-back))))
+
+(defun elfeed-sync--sort-response-entries (entries bad-feeds entries-by-title-date entries-by-url is-new)
+  (dolist (entry entries)
+    (setf (alist-get 'is-new entry) is-new)
+    (if (gethash (alist-get 'feed_url entry) bad-feeds)
+        (let ((title-date (format "%s---%s"
+                                  (alist-get 'title entry)
+                                  (alist-get 'updated entry))))
+          (puthash title-date entry entries-by-title-date))
+      (puthash (alist-get 'link entry) entry entries-by-url))))
+
+(defun elfeed-sync--get-response-entry (entry feed entries-by-title-date entries-by-url)
+  (if (gethash (or (elfeed-feed-url feed)
+                   (elfeed-feed-id feed))
+               bad-feeds)
+      (let ((title-date (format "%s---%s"
+                                (elfeed-entry-title entry)
+                                (format-time-string
+                                 "%Y-%m-%d %H:%M:%S"
+                                 (seconds-to-time (elfeed-entry-date entry))
+                                 "UTC0"))))
+        (prog1
+            (gethash title-date entries-by-title-date)
+          (remhash title-date entries-by-title-date)))
+    (prog1
+        (gethash (elfeed-entry-link entry) entries-by-url)
+      (remhash (elfeed-entry-link entry) entries-by-url))))
+
+(defun elfeed-sync--set-entry-unread (entry status)
+  "Set the unread status of ENTRY to STATUS.
+
+STATUS is a boolean.  If nil, the entry is marked as read. ENTRY is an instance of `elfeed-entry'."
+  (let ((is-unread (member elfeed-sync-unread-tag
+                           (elfeed-entry-tags entry))))
+    (if (and is-unread status)
+        (elfeed-untag entry elfeed-sync-unread-tag)
+      (when (not is-unread)
+        (elfeed-tag entry elfeed-sync-unread-tag)))))
+
+(defun elfeed-sync--set-entry-marked (entry status)
+  "Set the marked status of ENTRY to STATUS.
+
+STATUS is a boolean.  If nil, the entry is marked as
+unmarked.  ENTRY is an instance of `elfeed-entry'."
+  (let ((is-marked (member elfeed-sync-marked-tag
+                           (elfeed-entry-tags entry))))
+    (if (and is-marked status)
+        (elfeed-untag entry elfeed-sync-marked-tag)
+      (when (not is-marked)
+        (elfeed-tag entry elfeed-sync-marked-tag)))))
+
+(defun elfeed-sync--process-response (response bad-feeds)
+  (cl-loop for entry in (alist-get 'missing-entries response)
+           do (let* ((id (alist-get 'id entry))
+                     (attempts (or
+                                (gethash id (alist-get :missing elfeed-sync--state))
+                                0)))
+                (if (>= attempts elfeed-sync-max-retries)
+                    (puthash id (1+ attempts) (alist-get :missing elfeed-sync--state))
+                  (puthash id t (alist-get :discarded elfeed-sync--state))
+                  (remhash id (alist-get :missing elfeed-sync--state)))))
+  (setf (alist-get :missing-target elfeed-sync--state)
+        (seq-filter (lambda (datum)
+                      (< (- (time-convert nil 'integer) (car datum))
+                         elfeed-sync-missing-targets-keep))
+                    (alist-get :missing-target elfeed-sync--state)))
+  (let ((entries-by-title-date (make-hash-table :test #'equal))
+        (entries-by-url (make-hash-table :test #'equal)))
+    (elfeed-sync--sort-response-entries
+     (alist-get 'updated response) bad-feeds entries-by-title-date entries-by-url t)
+    (elfeed-sync--sort-response-entries
+     (mapcar #'cdr (alist-get :missing-target elfeed-sync--state))
+     bad-feeds entries-by-title-date entries-by-url nil)
+    (with-elfeed-db-visit (entry feed)
+      (when-let ((id (elfeed-ref-id (elfeed-entry-content entry))))
+        (if-let ((entry (elfeed-sync--get-response-entry entry feed entries-by-title-date entries-by-url)))
+            (progn
+              (elfeed-sync--set-entry-unread entry (alist-get 'unread entry))
+              (elfeed-sync--set-entry-marked entry (alist-get 'marked entry))
+              (puthash id (elfeed-sync-datum--create
+                           :id id
+                           :tags (elfeed-entry-tags entry))))
+          (unless (or (gethash id (alist-get :missing elfeed-sync--state))
+                      (gethash id (alist-get :discarded elfeed-sync--state)))
+            (puthash id (elfeed-sync-datum--create
+                         :id id
+                         :tags (elfeed-entry-tags entry))
+                     (gethash
+                      (or (elfeed-feed-url feed)
+                          (elfeed-feed-id feed))
+                      (alist-get :feeds elfeed-sync--state))))))
+      (when (> (- (time-convert nil 'integer)
+                  elfeed-sync-look-back)
+               (elfeed-entry-date entry))
+        (elfeed-db-return)))
+    (maphash (lambda (_ datum)
+               (when (alist-get 'is-new datum)
+                 (push datum (alist-get :missing-target elfeed-sync--state))))
+             entries-by-title-date)
+    (maphash (lambda (_ datum)
+               (when (alist-get 'is-new datum)
+                 (push datum (alist-get :missing-target elfeed-sync--state))))
+             entries-by-url))
+  (setf (alist-get :last-sync elfeed-sync--state)
+        (time-convert nil 'integer)))
+
+(defun elfeed-sync ()
+  (interactive)
+  (elfeed-sync--session
+   (let* ((bad-feeds (elfeed-sync--get-bad-feeds))
+          (request (elfeed-sync--prepare-request bad-feeds)))
+     (request (concat elfeed-sync-tt-rss-instance "/api/")
+       :type "POST"
+       :data (json-encode
+              `(("op" . "syncElfeed")
+                ("sid" . ,elfeed-sync--tt-rss-sid)
+                ("data" . ,request)))
+       :parser 'elfeed-sync--json-read-safe
+       :headers '(("Content-Type" . "application/json"))
+       :success (elfeed-sync--handler
+                 (elfeed-sync--process-response
+                  (alist-get 'content data)
+                  bad-feeds))))))
+
+;;;###autoload
+(define-minor-mode elfeed-sync-mode
+  "TODO"
+  :global t
+  (if elfeed-sync-mode
+      (progn
+        (add-hook 'kill-emacs-hook #'elfeed-sync-state-save-safe))
+    (remove-hook 'kill-emacs-hook #'elfeed-sync-state-save-safe)))
+
+
 (provide 'elfeed-sync)
 ;;; elfeed-sync.el ends here
diff --git a/init.php b/init.php
index ff84316..f8a02f4 100644
--- a/init.php
+++ b/init.php
@@ -14,6 +14,7 @@ class Elfeed_Sync extends Plugin {
 		$this->host = $host;
 
 		$this->host->add_api_method("setFeedsTree", $this);
+		$this->host->add_api_method("syncElfeed", $this);
 	}
 
 	function createCategoriesTree($tree, $feed_categories = null, $current_cat = null) {
@@ -113,9 +114,6 @@ class Elfeed_Sync extends Plugin {
 		return $results;
 	}
 
-	/**
-	 * Our own API.
-	 */
 	function setFeedsTree() {
 		$tree = $_REQUEST["tree"];
 
@@ -124,4 +122,124 @@ class Elfeed_Sync extends Plugin {
 
 		return array(API::STATUS_OK, array('subscribed' => $subscribed));
 	}
+
+	function updateEntries($data) {
+		$entries_query = ORM::for_table('ttrss_entries')
+				 ->table_alias('e')
+				 ->select_many('e.id', 'e.link', 'e.title', 'e.updated',
+							   'ue.marked', 'ue.unread', 'ue.feed_id')
+				 ->join('ttrss_user_entries', ['ue.ref_id', '=', 'e.id'], 'ue')
+				 ->join('ttrss_feeds', ['f.id', '=', 'ue.feed_id'], 'f')
+				 ->where('ue.owner_uid', $_SESSION['uid']);
+		if (!is_null($data['look_back'])) {
+			$entries_query = $entries_query->where_gte('updated', date('Y-m-d H:i:s', time() - $data['look_back']));
+		}
+		$entries = $entries_query->find_array();
+
+		$feeds = ORM::for_table('ttrss_feeds')
+			   ->select_many('id', 'feed_url')
+			   ->where('owner_uid', $_SESSION['uid'])
+			   ->find_array();
+
+		$feed_by_link = array();
+		foreach ($feeds as $feed) {
+			$feed_by_link[$feed['feed_url']] = $feed['id'];
+		}
+		$bad_feed_ids = array();
+		foreach($data['bad_feeds'] as $bad_feed_link) {
+			$feed_id = $feed_by_link[$bad_feed_link];
+			if ($feed_id) {
+				$bad_feed_ids[$feed_id] = true;
+			}
+		}
+
+		$entries_by_link = array();
+		$entries_by_title_date = array();
+		foreach ($entries as $entry) {
+			if ($bad_feed_ids[$entry['feed_id']]) {
+				$title_date = $entry['title'] . '---' . $entry['updated'];
+				$entries_by_title_date[$title_date] = $entry;
+			} else {
+				$entries_by_link[$entry['link']] = $entry;
+			}
+		}
+
+		$missing_feeds = array();
+		$missing_entries = array();
+		$toggle_unread = array();
+		$toggle_marked = array();
+		foreach($data['changed'] as $changed_entry) {
+			$feed_id = $feed_by_link[$changed_entry['feed_url']];
+			if (!$feed_id) {
+				$missing_feeds[$changed_entry['feed_url']] = true;
+				continue;
+			}
+			$bad_feed = $bad_feed_ids[$feed_id];
+			$entry = null;
+			if ($bad_feed) {
+				$title_date = $changed_entry['title'] . '---' . $changed_entry['date'];
+				$entry = $entries_by_title_date[$title_date];
+			} else {
+				$entry = $entries_by_link[$changed_entry['url']];
+			}
+			if (!$entry) {
+				array_push($missing_entries, $changed_entry);
+				continue;
+			}
+
+			if (!is_array($changed_entry['tags'])) {
+				continue;
+			}
+			$unread = in_array($data['unread_tag'], $changed_entry['tags']);
+			$marked = in_array($data['starred_tag'], $changed_entry['tags']);
+			if ($entry['unread'] != $unread) {
+				array_push($toggle_unread, $entry['id']);
+			}
+			if ($entry['marked'] != $marked) {
+				array_push($toggle_marked, $entry['id']);
+			}
+		}
+
+		ORM::get_db()->beginTransaction();
+		if (count($toggle_unread) > 0) {
+			ORM::raw_execute('UPDATE ttrss_user_entries t SET unread = not t.unread WHERE ref_id IN ('.implode(',', $toggle_unread).')');
+		}
+		if (count($toggle_marked) > 0) {
+			ORM::raw_execute('UPDATE ttrss_user_entries t SET marked = not t.marked WHERE ref_id IN ('.implode(',', $toggle_marked).')');
+		}
+		ORM::get_db()->commit();
+
+		return array($missing_feeds, $missing_entries, sizeof($toggle_unread), sizeof($toggle_marked));
+	}
+
+	function getUpdatedEntries($data) {
+		if (is_null($data['last_sync'])) {
+			return [];
+		}
+		$last_sync = date('Y-m-d H:i:s', $data['last_sync']);
+		$entries = ORM::for_table('ttrss_entries')
+				 ->select_many('link', 'ttrss_entries.title', 'updated', 'feed_url', 'marked', 'unread')
+				 ->join('ttrss_user_entries', ['ttrss_entries.id', '=', 'ttrss_user_entries.ref_id'])
+				 ->join('ttrss_feeds', ['ttrss_feeds.id', '=', 'ttrss_user_entries.feed_id'])
+				 ->where_raw('ttrss_user_entries.last_read >= ? OR ttrss_user_entries.last_marked >= ?', [$last_sync, $last_sync])
+				 ->where('ttrss_user_entries.owner_uid', $_SESSION['uid'])
+				 ->find_array();
+		return $entries;
+	}
+
+	function syncElfeed() {
+		$data = $_REQUEST["data"];
+
+		list($missing_feeds, $missing_entries, $toggled_unread, $toggled_marked) = $this->updateEntries($data);
+		$changed_entries = $this->getUpdatedEntries($data);
+		return array(API::STATUS_OK,
+					 array(
+						 'missing_feeds' => $missing_feeds,
+						 'missing_entries' => $missing_entries,
+						 'toggled_unread' => $toggled_unread,
+						 'toggled_marked' => $toggled_marked,
+						 'updated' => $changed_entries
+					 )
+		);
+	}
 }