Merge pull request #9 from SqrtMinusOne/update-data

Update API data, add biome-multi-history
This commit is contained in:
Pavel Korytov 2024-04-26 17:07:48 +03:00 committed by GitHub
commit 804a0576f5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 5434 additions and 1250 deletions

File diff suppressed because it is too large Load diff

View file

@ -36,16 +36,19 @@
(defvar biome-api-parse--data nil
"Data parsed from the API docs.")
(defvar biome-api-parse--htmls (make-hash-table :test #'equal)
"HTML strings of the API docs.")
(defconst biome-api-parse--urls
'(((:name . "Weather Forecast")
(:url . "https://open-meteo.com/en/docs")
(:description . "Seamless integration of high-resolution weather models with up 16 days forecast")
(:key . "ww"))
((:name . "DWD ICON")
((:name . "DWD ICON (Germany)")
(:url . "https://open-meteo.com/en/docs/dwd-api")
(:description . "German Weather Service ICON model. 15-minutely data for Central Europe")
(:key . "wd"))
((:name . "NOAA GFS & HRRR")
((:name . "NOAA GFS & HRRR (U.S.)")
(:url . "https://open-meteo.com/en/docs/gfs-api")
(:description . "Forecasts tailored for the US region")
(:key . "wu"))
@ -57,18 +60,26 @@
(:url . "https://open-meteo.com/en/docs/ecmwf-api")
(:description . "Open-data forecasts by ECMWF")
(:key . "we"))
((:name . "JMA")
((:name . "JMA (Japan)")
(:url . "https://open-meteo.com/en/docs/jma-api")
(:description . "Forecasts tailored for Japan")
(:key . "wj"))
((:name . "MET Norway")
((:name . "MET (Norway)")
(:url . "https://open-meteo.com/en/docs/metno-api")
(:description . "Forecasts exclusively for North Europe")
(:key . "wn"))
((:name . "GEM")
((:name . "GEM (Canada)")
(:url . "https://open-meteo.com/en/docs/gem-api")
(:description . "Forecasts tailored for North America")
(:key . "wa"))
((:name . "BOM (Australia)")
(:url . "https://open-meteo.com/en/docs/bom-api/")
(:description . "Weather forecasts from the Australian Bureau of Meteorology")
(:key . "wb"))
((:name . "CMA (China)")
(:url . "https://open-meteo.com/en/docs/gem-api")
(:description . "Weather forecasts from the Chinese Meteorological Administration")
(:key . "wc"))
((:name . "Historical Weather")
(:url . "https://open-meteo.com/en/docs/historical-weather-api")
(:description . "Weather information since 1940")
@ -77,6 +88,10 @@
(:url . "https://open-meteo.com/en/docs/ensemble-api")
(:description . "Weather information since 1940")
(:key . "e"))
((:name . "Previous Runs API")
(:url . "https://open-meteo.com/en/docs/previous-runs-api")
(:description . "Weather Forecasts from Previous Days to Compare Run-To-Run Performance")
(:key . "v"))
((:name . "Climate Change")
(:url . "https://open-meteo.com/en/docs/climate-api")
(:description . "Climate change projections")
@ -100,6 +115,7 @@
("Hourly Weather Variables" . "hourly")
("15-Minutely Weather Variables" . "minutely_15")
("3-Hourly Weather Variables" . "hourly")
("Current Weather" . "current")
("Hourly Marine Variables" . "hourly")
("Daily Marine Variables" . "daily")
("Hourly Air Quality Variables" . "hourly")
@ -137,11 +153,61 @@
"Replace these variable defintions with the given ones.")
(defconst biome-api-parse--add-settings
`(((:param . ("elevation" . ((:name . "Elevation")
`(("settings" . (((:param . ("elevation" . ((:name . "Elevation")
(:type . float))))
(:pages . ("Weather Forecast" "DWD ICON" "NOAA GFS & HRRR"
"MeteoFrance" "ECMWF" "JMA" "MET Norway" "GEM" "Historical Weather"
(:pages . ("Weather Forecast"
"DWD ICON (Germany)"
"NOAA GFS & HRRR (U.S.)"
"MeteoFrance"
"ECMWF"
"JMA (Japan)"
"MET (Norway)"
"GEM (Canada)"
"BOM (Australia)"
"CMA (China)"
"Historical Weather"
"Ensemble Models")))))
("coordinates and time" . (((:param . ("start_date" . ((:name . "Start date")
(:type . date))))
(:pages . ("Weather Forecast"
"DWD ICON (Germany)"
"NOAA GFS & HRRR (U.S.)"
"MeteoFrance"
"ECMWF"
"JMA (Japan)"
"MET (Norway)"
"GEM (Canada)"
"BOM (Australia)"
"CMA (China)"
"Ensemble Models"
"Previous Runs API"
"Marine Forecast"
"Flood")))
((:param . ("end_date" . ((:name . "End date")
(:type . date))))
(:pages . ("Weather Forecast"
"DWD ICON (Germany)"
"NOAA GFS & HRRR (U.S.)"
"MeteoFrance"
"ECMWF"
"JMA (Japan)"
"MET (Norway)"
"GEM (Canada)"
"BOM (Australia)"
"CMA (China)"
"Ensemble Models"
"Previous Runs API"
"Marine Forecast"
"Flood")))))))
;; TODO make empty when the website is fixed
(defconst biome-api-parse--field-names-default
'(("tilt" . "Panel Tilt (0° horizontal)")
("azimuth" . "Panel Azimuth (0° S, -90° E, 90° W)")))
;; TODO make empty when the website is fixed
(defconst biome-api-parse--fix-ids
'(("past_minutely_15s" . "past_minutely_15")))
(defun biome-api-parse--fix-string (string)
"Remove extra spaces and newlines from STRING."
@ -151,27 +217,47 @@
" "
string)))
(defun biome-api-parse--table-variables (section)
"Parse variable in table in SECTION.
SECTION is a DOM element. Return a list of sections as defined by
`biome-api-parse--page', one section per each table row."
(when-let* ((table (car (dom-by-class section "table-responsive")))
(table-rows (dom-by-tag table 'tr)))
(cl-loop for row in table-rows
for name = (dom-text (car (dom-by-tag row 'td)))
for fields = (biome-api-parse--page-variables row)
;; do (cl-loop for field in fields
;; do (setf (alist-get :name field)
;; (format "%s (%s)" (alist-get :name field) name)))
collect `((:name . ,name)
(:fields . ,fields)))))
(defun biome-api-parse--page-variables (section)
"Parse variables from SECTION.
SECTION is a DOM element. Return a list of fields as defined by
`biome-api-parse--page'."
(let ((elements (dom-search section (lambda (el) (not (stringp el)))))
fields field-names field-id-mapping)
(field-names (copy-tree biome-api-parse--field-names-default))
fields field-id-mapping)
(cl-loop for elem in elements
for dom-id = (seq-some
(lambda (v) (unless (string-empty-p v) v))
(list (dom-attr elem 'id)
(dom-attr elem 'name)))
for id = (seq-some
for id = (let ((id (seq-some
(lambda (v) (unless (string-empty-p v) v))
(list (let ((val (dom-attr elem 'value)))
(unless (member val '("true" "false")) val))
dom-id))
dom-id))))
(alist-get id biome-api-parse--fix-ids
id nil #'equal))
if (and (member (dom-tag elem) '(input select))
(not (equal id dom-id)))
do (push (cons dom-id id) field-id-mapping)
if (member id biome-api-parse--exclude-ids)
if (or (member id biome-api-parse--exclude-ids)
(equal (dom-attr elem 'type) "hidden"))
do (null nil) ; how to do nothing? :D
else if (member id biome-api-parse--float-ids)
do (push (cons id '((:type . float))) fields)
@ -183,11 +269,15 @@ SECTION is a DOM element. Return a list of fields as defined by
do (push
(cons id
`((:type . select)
(:options . ,(mapcar
(:options . ,(seq-filter
(lambda (item)
(not (equal (cdr item)
"- (default)")))
(mapcar
(lambda (opt) (cons (dom-attr opt 'value)
(biome-api-parse--fix-string
(dom-texts opt))))
(dom-by-tag elem 'option)))))
(dom-by-tag elem 'option))))))
fields)
else if (eq (dom-tag elem) 'label)
do (push (cons (or (cdr (assoc (dom-attr elem 'for) field-id-mapping))
@ -244,6 +334,10 @@ Return a list of sections as defined by `biome-api-parse--page'."
(push `((:name . ,item-name)
(:children . ,(biome-api-parse--page-pills item)))
res)
else if (dom-by-class item "table-responsive") do
(push `((:name . ,item-name)
(:children . ,(biome-api-parse--table-variables item)))
res)
else do (push `((:name . ,item-name)
(:fields . ,(biome-api-parse--page-variables item)))
res))
@ -255,6 +349,8 @@ Return a list of sections as defined by `biome-api-parse--page'."
The return value is as defined by `biome-api-parse--page'."
(or (when-let ((accordion (biome-api-parse--page-accordion section)))
`((:children . ,accordion)))
(when-let ((table (biome-api-parse--table-variables section)))
`((:children . ,table)))
(when-let ((variables (biome-api-parse--page-variables section)))
`((:fields . ,variables)))))
@ -306,23 +402,39 @@ NAME is the page name as given in `biome-api-parse--urls'."
,@data)))
;; Extract the model section from the hourly accordion and add it to
;; the root section.
(when-let ((models-data (biome-api-parse--postprocess-extract-section
sections "models" t)))
(when-let* ((models-data (biome-api-parse--postprocess-extract-section
sections "models" t))
(models-section (cdr models-data)))
(setq sections (append (car models-data)
(list (cdr models-data)))))
(list models-section))))
;; Likewise with the 15-minutely section.
(when-let* ((15-minutely-data (biome-api-parse--postprocess-extract-section
sections "15-minutely" t))
(15-minutely-section (cdr 15-minutely-data)))
(setq sections (append (car 15-minutely-data)
(list 15-minutely-section))))
(when-let ((location-data (biome-api-parse--postprocess-extract-section
sections "location and time" t)))
(setf (alist-get :name location-data)
"Select Coordinates and Time"))
;; Add settings
(when-let ((settings-data (biome-api-parse--postprocess-extract-section
sections "settings")))
(cl-loop for var in biome-api-parse--add-settings
(cl-loop
for (section-name . add-vars) in biome-api-parse--add-settings
do (when-let ((settings-data (biome-api-parse--postprocess-extract-section
sections section-name)))
(cl-loop for var in add-vars
if (member name (alist-get :pages var))
do (push (copy-tree (alist-get :param var))
(alist-get :fields (cdr settings-data))))
do (setf (alist-get :fields (cdr settings-data))
(cons (copy-tree (alist-get :param var))
(alist-get :fields (cdr settings-data)))))
;; Fix forecast_days for Flood API
(when (equal name "Flood")
(let ((forecast-days (alist-get "forecast_days"
(when-let ((forecast-days (alist-get "forecast_days"
(alist-get :fields (cdr settings-data))
nil nil #'equal)))
(setf (alist-get :max forecast-days) 210))))
(setf (alist-get :max forecast-days) 210)))))
;; Add section-specific URL params
;; XXX I do not know why this doesn't work without returning
;; sections from the loop
@ -403,8 +515,14 @@ fields attributes as cdr:
"Parse one page from open-meteo API docs.
DATUM is an element of `biome-api-parse--urls'."
(let* ((html (or (gethash (alist-get :url datum) biome-api-parse--htmls)
(progn
(browse-url (alist-get :url datum))
(let* ((html (read-string "Enter HTML string: "))
(let ((html (read-string "Enter HTML string: ")))
(puthash (alist-get :url datum) html
biome-api-parse--htmls)
html))))
(parsed (biome-api-parse--page html (alist-get :name datum))))
(setf (alist-get (alist-get :name datum)
biome-api-parse--data nil nil #'equal)
@ -421,7 +539,7 @@ DATUM is an element of `biome-api-parse--urls'."
seq-uniq
(seq-sort #'string-lessp)))
(defun biome-api-parse--generate ()
(defun biome-api-parse--generate (&optional arg)
"Generate `biome-api-data' and `biome-api-timezones' constants.
This function does two things:
@ -432,14 +550,20 @@ Unfortunately, the HTML pages have accordions that are dynamically
loaded, so we need to manually load them in the browser, expand the
accordions and copy the HTML source.
HTML strings are cached in `biome-api-parse--htmls'. Set ARG to
non-nil to reset.
The function prints the generated constants to a new buffer. Save
them to biome-api-data.el."
(interactive)
(interactive "P")
(setq biome-api-parse--data nil)
(when arg
(setq biome-api-parse--htmls (make-hash-table :test #'equal)))
(let ((timezones (biome-api-parse--timezones)))
(cl-loop for datum in biome-api-parse--urls
do (biome-api-parse--datum datum))
(let ((buffer (generate-new-buffer "*biome-generated*")))
(let ((buffer (generate-new-buffer "*biome-generated*"))
(indent-tabs-mode nil))
(with-current-buffer buffer
(emacs-lisp-mode)
(insert (pp-to-string

View file

@ -30,15 +30,20 @@
(defconst biome-api--default-urls
'(("Weather Forecast" . "https://api.open-meteo.com/v1/forecast")
("DWD ICON" . "https://api.open-meteo.com/v1/dwd-icon")
("NOAA GFS & HRRR" . "https://api.open-meteo.com/v1/gfs")
("DWD ICON (Germany)" . "https://api.open-meteo.com/v1/dwd-icon")
("NOAA GFS & HRRR (U.S.)" . "https://api.open-meteo.com/v1/gfs")
("MeteoFrance" . "https://api.open-meteo.com/v1/meteofrance")
("ECMWF" . "https://api.open-meteo.com/v1/ecmwf")
("JMA" . "https://api.open-meteo.com/v1/jma")
("MET Norway" . "https://api.open-meteo.com/v1/metno")
("GEM" . "https://api.open-meteo.com/v1/gem")
("JMA (Japan)" . "https://api.open-meteo.com/v1/jma")
("MET (Norway)" . "https://api.open-meteo.com/v1/metno")
("GEM (Canada)" . "https://api.open-meteo.com/v1/gem")
("BOM (Australia)" . "https://api.open-meteo.com/v1/bom")
("CMA (China)" . "https://api.open-meteo.com/v1/cma")
("Historical Weather" . "https://archive-api.open-meteo.com/v1/archive")
("Historical Weather (on this day)"
. "https://archive-api.open-meteo.com/v1/archive")
("Ensemble Models" . "https://ensemble-api.open-meteo.com/v1/ensemble")
("Previous Runs API" . "https://previous-runs-api.open-meteo.com/v1/forecast")
("Climate Change" . "https://climate-api.open-meteo.com/v1/climate")
("Marine Forecast" . "https://marine-api.open-meteo.com/v1/marine")
("Air Quality" . "https://air-quality-api.open-meteo.com/v1/air-quality")

View file

@ -499,6 +499,9 @@ column.
ENTRIES is a list of values. COL-KEY is the column key as given by
the API. UNIT is the unit of the column."
;; Current weather is not a sequence.
(unless (sequencep entries)
(setq entries (list entries)))
(let ((format-def (biome-grid--get-format-def col-key unit)))
(mapcar
(lambda (entry)
@ -709,7 +712,8 @@ by `biome-api-get')."
(defun biome-grid--maybe-highlight-current ()
"Highlight current hour or day (if hour is not found)."
(when biome-grid-highlight-current
(when (and biome-grid-highlight-current
(length> tabulated-list-entries 1))
(save-excursion
(goto-char (point-min))
(let ((needle-datetime

View file

@ -29,6 +29,7 @@
(require 'transient)
(require 'biome-query)
(require 'biome-api-parse)
;; XXX Recursive imports T_T
(declare-function biome-preset "biome")
@ -287,5 +288,108 @@ as it is necessary for `biome-grid'."
unit)))))
(multi . ,(biome-multi--join-results queries query-names vars-mapping results))))))
(defun biome-multi--history-section ()
"Create a section for `biome-multi-history'.
This is based on the Historical Weather section."
(let* ((history-params
(copy-tree (alist-get "Historical Weather" biome-api-data
nil nil #'equal)))
(time-section (biome-api-parse--postprocess-extract-section
(alist-get :sections history-params)
"coordinates and time"))
(current-year (decoded-time-year (decode-time))))
(push '("day_of_year" . ((:name . "Day of Year")
(:type . date)))
(alist-get :fields time-section))
(push
`("end_year" . ((:name . "End year")
(:type . number)
(:min . 1940)
(:max . ,current-year)))
(alist-get :fields time-section))
(push
`("start_year" . ((:name . "Start year")
(:type . number)
(:min . 1940)
(:max . ,current-year)))
(alist-get :fields time-section))
(setf (alist-get :name history-params)
"Historical Weather (on this day)")
(setf (alist-get :fields time-section)
(seq-filter
(lambda (elem)
(not (member (car elem) '("start_date" "end_date"))))
(alist-get :fields time-section)))
history-params))
(defun biome-multi-history--prepare-queries (query)
"Create queries for `biome-multi-history'.
QUERY is a query as defined by `biome-query-current', prepared like
for the normal Historical Weather section but with the following
added fields:
- start_year (number from 1940 to current)
- end_year (number from 1940 to current)
- day_of_year (timestamp)."
(let ((start-year (alist-get "start_year"
(alist-get :params query)
nil nil #'equal))
(end-year (alist-get "end_year"
(alist-get :params query)
nil nil #'equal))
(day-of-year (alist-get "day_of_year"
(alist-get :params query)
nil nil #'equal)))
(unless (and start-year end-year day-of-year)
(user-error "Set Start Year, End Year and Day of Year"))
(cl-loop with current-date = (decode-time (seconds-to-time day-of-year))
for year from start-year to end-year
for date = (copy-tree current-date)
do (setf (decoded-time-year date) year)
for time = (time-convert (encode-time date) 'integer)
for year-query = (copy-tree query)
do (setf (alist-get :params year-query)
(seq-filter (lambda (elem)
(not
(member
(car elem)
'("day_of_year" "end_year" "start_year"))))
(alist-get :params year-query)))
do (push (cons "start_date" (- time (% time (* 60 60 24))))
(alist-get :params year-query))
do (push (cons "end_date" (- time (% time (* 60 60 24))))
(alist-get :params year-query))
collect year-query)))
(defun biome-multi--history-query (callback)
"Get historical weather data on a particular day.
CALLBACK is called with a list of queries, one per day."
(interactive (list nil))
(let ((params (biome-multi--history-section)))
(setq biome-query--callback
(lambda (query)
(let ((queries (biome-multi-history--prepare-queries query)))
(when (y-or-n-p (format "Send %s requests to the API?"
(length queries)))
(funcall callback queries)))))
(biome-query--section-open-params params)))
(defun biome-multi--concat-results (queries results)
"Concat RESULTS from multiple Open Meteo responses.
QUERIES is a list of forms as defined by `biome-query-current'. Each
query is assumed to have the same variables. RESULTS is a list of
responses from Open Meteo."
(let ((group (intern (alist-get :group (car queries)))))
(cl-loop for result in (cdr results)
do (cl-loop
for (var . values) in (alist-get group result)
do (setf (alist-get var (alist-get group (car results)))
(vconcat (alist-get var (alist-get group (car results)))
values))))
(car results)))
(provide 'biome-multi)
;;; biome-multi.el ends here

View file

@ -41,7 +41,8 @@
;; XXX Recursive imports T_T
(declare-function biome-preset "biome")
(declare-function biome-multi "biome")
(declare-function biome-multi "biome-multi")
(declare-function biome-multi-history "biome-multi")
(defcustom biome-query-max-fields-in-row 20
"Maximum number of fields in a row."
@ -89,7 +90,10 @@ The format is: (name latitude longitude)."
:type 'string
:group 'biome)
(defconst biome-query-groups '("daily" "hourly" "minutely_15" "hourly")
(defconst biome-query--max-sections-for-row 6
"Maximum number of sections to use for `transient-row'.")
(defconst biome-query-groups '("daily" "hourly" "minutely_15" "hourly" "current")
"Name of groups.
A group is a mutually exclusive choice. E.g. in the \"Weather
@ -100,7 +104,9 @@ have to be displayed separately.")
(defconst biome-query--split-items '(("timezone" . "time zone")
("timeformat" . "time format")
("weathercode" . "weather code")
("iso8601" . "iso 8"))
("iso8601" . "iso 8")
;; I'm used to "c" for "coordinates"
("current weather" . "urrent weather"))
"Items to split into separate words for generating keys.")
(defconst biome-query--ignore-items '("m" "cm")
@ -204,7 +210,7 @@ QUERY is a form as defined by `transient-define-prefix'."
(setq lat (cdr item)))
((equal (car item) "longitude")
(setq lon (cdr item)))
((member (car item) '("end_date" "start_date"))
((member (car item) '("end_date" "start_date" "day_of_year"))
(push
(format "%s: %s" (propertize
(biome-query--get-header (car item) var-names)
@ -850,7 +856,9 @@ the position of the current section in the `biome-api-data' tree."
,@(thread-last
(append
fields
(when (equal (alist-get :name (car parents)) "Select Coordinates or City")
(when (string-match-p
(rx "Select Coordinates")
(alist-get :name (car parents)))
'(coords)))
(seq-map-indexed
(lambda (field idx) (cons field (/ idx biome-query-max-fields-in-row))))
@ -877,7 +885,9 @@ KEYS is the result of `biome-query--unique-keys'. PARENTS is a
list of parent sections."
(when sections
`(["Sections"
:class transient-row
:class ,(if (length> sections biome-query--max-sections-for-row)
'transient-column
'transient-row)
,@(mapcar
(lambda (section)
`(,(gethash (alist-get :name section) keys)
@ -1018,18 +1028,27 @@ SECTION is a form as defined in `biome-api-parse--page'."
(:parents . ,parents))))
(put 'biome-query-section 'transient--layout nil)))
(defun biome-query--section-open (name)
"Open section NAME in `biome-query--section'."
(let ((params (alist-get name biome-api-data nil nil #'equal)))
(unless params
(error "No such section: %s" name))
(defun biome-query--section-open-params (params)
"Open section defined by PARAMS in `biome-query--section'.
PARAMS is a form as defiend by `biome-api-data'."
(setq biome-query--current-section params)
(when (and biome-query-current
(not (equal name (alist-get :name biome-query-current))))
(not (equal (alist-get :name params)
(alist-get :name biome-query-current))))
(setq biome-query-current nil))
(unless biome-query-current
(biome-query--reset-report))
(funcall-interactively #'biome-query--section params)))
(funcall-interactively #'biome-query--section params))
(defun biome-query--section-open (name)
"Open section NAME in `biome-query--section'."
(let ((params (alist-get name biome-api-data nil nil #'equal)))
(cond
((equal name "Historical Weather (on this day)")
(biome-multi-history))
(params (biome-query--section-open-params params))
(t (error "No such section: %s" name)))))
(transient-define-prefix biome-query (callback)
["Open Meteo Data"
@ -1044,11 +1063,15 @@ SECTION is a form as defined in `biome-api-parse--page'."
(lambda () (interactive)
(biome-query--section-open ,name))
:transient transient--do-stack))))]
["Aggregate Data"
:class transient-column
("i" "Historical Weather (on this day)" biome-multi-history
:transient transient--do-stack)
("u" "Join multiple queries" biome-multi :transient transient--do-stack)]
["Actions"
:class transient-row
("r" "Resume" biome-resume :transient transient--do-replace)
("r" "Resume" biome-resume :transient transient--do-stack)
("p" "Preset" biome-preset :transient transient--do-stack)
("u" "Join multiple queries" biome-multi :transient transient--do-stack)
("q" "Quit" transient-quit-one)]
(interactive (list nil))
(unless callback

View file

@ -4,7 +4,7 @@
;; Author: Korytov Pavel <thexcloud@gmail.com>
;; Maintainer: Korytov Pavel <thexcloud@gmail.com>
;; Version: 0.1.0
;; Version: 0.3.0
;; Package-Requires: ((emacs "27.1") (transient "0.3.7") (ct "0.2") (request "0.3.3") (compat "29.1.4.1"))
;; Homepage: https://github.com/SqrtMinusOne/biome
;; Published-At: 2023-07-22
@ -106,6 +106,18 @@ preset definition\" in `biome' or `biome-multi'."
(let ((merged (biome-multi--merge queries results)))
(funcall biome-frontend (nth 0 merged) (nth 1 merged))))))))
(defun biome-multi-history ()
"Get historical weather data on a particular day."
(interactive)
(funcall-interactively
#'biome-multi--history-query
(lambda (queries)
(biome-api-get-multiple
queries
(lambda (queries results)
(let ((concat-results (biome-multi--concat-results queries results)))
(funcall biome-frontend (car queries) concat-results)))))))
(defmacro biome-def-preset (name params)
"Declare a query preset.