Shared CCE Helpers

Tags:  CCE

This constitutes the backbone of The Complete Computing Environment. These functions rely on #+KEYWORD style metadata embedded throughout my org-roam Knowledge Base and some custom extensions to org-roam to enable this keyword caching. Taken together, these functions provide a concrete API (or at the inspiration for a generic API) for building limited Literate Programming applications, org-mode meta applications which can answer interesting questions from across your knowledge base. It's unfortunate that this relies on patches to org-roam which aren't upstreamed, I'm awaiting a plugin architecture that lets me decouple this. In the meantime my home-manager installs the version of org-roam which has these additions. I've constructed an Ouroboros.

Module API

Here's what matters:

  • the variable cce-directory can be used where necessary.
  • cce/module-list which returns a big old blob of data, a list of modules the system has cached, sorted by the CCE_PRIORITY keyword value: (prin1-to-string (first (cce/module-list))) {{{results(("configure_packaging.org" ("CCE_PRIORITY" . "00") ("CCE_ANSIBLE" . "packaging") ("CCE_MODULE" . "packaging") ("CCE_PREDICATE" . "t")))}}}
  • cce/generate-dynamic-init and cce/generate-dynamic-ansible do what they say on the tin: create an init.el or main.yml with the tangled files inserted in to them in the order returned by the module-list.
(provide 'cce/cce-common)
(require 'cl)

(setq cce-directory "~/org/cce/")
(setq cce-dynamic-init (concat cce-directory "/init.el"))
(setq cce-dynamic-ansible-main "~/org/cce/roles/endpoint/tasks/main.yml")

(defun cce/get-filename-from-link (descr)
  (car (last
        (split-string
         (replace-regexp-in-string org-link-bracket-re "\\1" (first descr))
         ":"))))

(defun cce/fetch-keywords-from-cache ()
  (let ((keywords (apply #'vector
                         (seq-filter (lambda (f)
                                       (s-starts-with-p "CCE_" f))
                                     org-roam-cached-keywords))))
    (mapcar (lambda (row)
              (cons (file-relative-name (car row))
                    (cdr row)))
            (org-roam-db-query
             [:select [file keyword value] :from keywords
              :where (in keyword $v1)] keywords))))

(defun cce/fetch-titles-from-cache ()
  (let ((collected (org-roam-db-query   
                    [:select [file title] :from titles
                     :where (in file $v1)
                     :group-by file] 
                    (apply #'vector (mapcar (lambda (mod)
                                              (expand-file-name (car mod)))
                                            (cce/module-list))))))
    (when (= 0 (length collected))
      (error "No titles returned from cache."))
    (mapcar (lambda (row)
              (let ((file (file-relative-name (car row)))
                    (title (cadr row))) ; always take first title
                (list file "TITLE" title)))
            collected)))

(use-package s)
(defun cce/collect-modules-from-roam (&optional include-titles)
  (let* ((ht (make-hash-table :test 'equal))
         (results (cce/fetch-keywords-from-cache)))
    (mapc (lambda (row)
            (let* ((file (s-chop-prefix org-roam-directory (first row)))
                   (filerows (gethash file ht)))
              (puthash (first row)
                       (append filerows `((,(second row) . ,(third row))))
                       ht)))
          (if include-titles (append results (cce/fetch-titles-from-cache)) results))
    ht))

(defun cce/module-list (&optional include-titles)
  "returns a list of CCE modules sorted by their priority. CAR of each element is the file name, CDR is an assocation list"
  (let* ((ht (cce/collect-modules-from-roam include-titles))
         (modules
          (mapcar #'cadr
                  (sort
                   (mapcar (lambda (key)
                             (let ((md (gethash key ht)))
                               (list (alist-get "CCE_PRIORITY" md "90" nil #'equal)
                                     (cons key md))))
                           (hash-table-keys ht))
                   (lambda (one two) (string-lessp (first one) (first two)))))))
    (if (not (= 0 (length modules)))
        modules
      (error "Nothing returned from cache."))))

(defun cce/make-module-file-name (module)
  (concat cce-directory module ".el"))

(defun cce/generate-dynamic-init (module-list)
  "Returns an init buffer of the processed MODULE-LIST as taken from [`cce/module-list']"
  (with-temp-file cce-dynamic-init
    (insert ";; -*- lexical-binding: t -*-\n")
    (mapc (lambda (module)
            (when-let* ((file (car module))
                        (meta (cdr module))
                        (module (alist-get "CCE_MODULE" meta nil nil 'equal)))
              (if-let* ((pred-str (alist-get "CCE_PREDICATE" meta nil nil 'equal))
                        (predicate (read (or pred-str "nil"))))
                  (when-let* ((val (eval predicate)))
                    (goto-char (point-max))
                    (insert-file-contents (cce/make-module-file-name module)))
                (progn
                  (goto-char (point-max))
                  (insert-file-contents (cce/make-module-file-name module))))))
          module-list)))

(defun cce/generate-dynamic-ansible (module-list)
  "Returns a main.yml buffer of the processed MODULE-LIST as taken from [`cce/module-list']"
  (with-temp-file cce-dynamic-ansible-main
    (mapc (lambda (module)
            (when-let* ((file (car module))
                        (meta (cdr module))
                        (module (alist-get "CCE_ANSIBLE" meta nil nil 'equal)))
              (if-let* ((pred-str (alist-get "CCE_PREDICATE" meta nil nil 'equal))
                        (predicate (read (or pred-str "nil"))))
                  (when-let* ((val (eval predicate)))
                    (goto-char (point-max))
                    (insert (format "- import_tasks: %s.yml\n" module)))
                (progn
                  (goto-char (point-max))
                  (insert (format "- import_tasks: %s.yml\n" module))))))
          module-list)))
(setq cce-nixos-directory "~/org/cce/nixlib")

(defun alist-get-alike (key alist &optional default remove)
  (alist-get key alist default remove #'equal))

(defun cce/split-cce-nixos-string (modules)
  "CCE_HOME_MODULE can have options, pull them out. if i add one more i need to just make these their own keywords..."
  (->> modules 
    (seq-filter (lambda (module) (->> module
                                   (cdr)
                                   (alist-get-alike "CCE_HOME_MODULE"))))
    (seq-map (lambda (module)
               (let* ((subs (->> module
                              (cdr)
                              (alist-get-alike "CCE_HOME_MODULE")
                              (split-string)))
                      (with-dups (append module `(("CCE_HOME_MODULE" . ,(first subs)) ("CCE_HOME_PACKAGES" . ,(second subs))))))
                 (cl-remove "CCE_HOME_MODULE" with-dups :key #'car :start 1 :count 1 :test #'equal))))))

(defun cce/compile-nixos-statements (modules)
  "loads the nix files for each module and returns a string of them catted together"
  (->> modules
    (seq-remove (lambda (module)
                  (let ((hm (alist-get-alike "CCE_HOME_MODULE" (cdr module)) ))
                    (or (not hm)
                        (equal hm "nil")))))
    (seq-map (lambda (module)
               (with-temp-buffer
                 (insert-file-contents (expand-file-name (alist-get-alike "CCE_HOME_MODULE" (cdr module))
                                                         cce-directory)) 
                 (buffer-string))))
    (apply #'concat)))

(defun cce/home-manager-transclude-nix-files ()
  (->> (cce/module-list)
       (cce/split-cce-nixos-string)
       (cce/compile-nixos-statements)))
(defun cce/home-manager-custom-derivations ()
  ""
  (let ((cat-strs (lambda (strings) (string-join strings " ++ "))))
    (->> (cce/module-list)
         (cce/split-cce-nixos-string)
         (seq-map (lambda (item) (alist-get-alike "CCE_HOME_DERIVATIONS" (cdr item))))
         (seq-filter #'identity)
         (seq-map (lambda (dfile)
                    (with-temp-buffer
                      (insert-file-contents (expand-file-name dfile cce-directory)) 
                      (buffer-string))))
         (apply #'concat))))
(defun cce/home-manager-collect-epkg-overrides ()
  (->> (cce/module-list)
    (seq-map (lambda (item) (alist-get-alike "CCE_HOME_EPKGS" (cdr item))))
    (seq-filter #'identity)
    (seq-map (lambda (dfile)
               (with-temp-buffer
                 (insert-file-contents (expand-file-name dfile cce-directory)) 
                 (buffer-string))))
    (apply #'concat)))

This isn't used any more since I'm pulling the package lists out dynamically:

(defun cce/home-manager-collect-packages ()
  ""
  (let ((cat-strs (lambda (strings) (string-join strings " ++ "))))
    (->> (cce/module-list)
         (cce/split-cce-nixos-string)
         (seq-map (lambda (item) (alist-get-alike "CCE_HOME_PACKAGES" (cdr item))))
         (seq-filter #'identity)
         (seq-map (lambda (package) (format "lib.mypkgs.%s" package)))
         (funcall cat-strs)
         (format "home.packages = %s;\n"))))

Invoking the CCE

These interactive commands invoke the "dynamic loaders" used in the CCE index to generate and execute the init.el and Ansible files:

(defun cce/dynamic-ansible (&optional tags roles) 
  (interactive)
  (let ((modules (cce/module-list))
        (command (format "bash %s %s"
                         (expand-file-name "cce.shell" cce-directory)
                         (if tags (concat "-t " tags) ""))))
    (cce/generate-dynamic-ansible modules)
    (async-shell-command command "*cce-install*")))


(defun cce/dynamic-init () 
  (interactive)
  (let ((modules (cce/module-list)))
    (cce/generate-dynamic-init modules)
    (copy-file "~/org/cce/init.el" "~/.emacs.d/init.el" t t)
    (delete-file "~/.emacs.d/init.elc")))

Predicate Functions

These are simple predicate functions which can be used in the CCE_PREDICATE:

(defun cce/using-termux ()
  "A CCE predicate"
  (or (not (string-blank-p (or (getenv "ANDROID_DATA") "")))
      (and (boundp 'building-for-termux)
           building-for-termux)))

(defun cce/has-display-frames ()
  "Dynamic CCE predicate"
  (and (not (cce/using-termux))
       (> (length (frame-list)) 0)))

(defun cce/using-exwm ()
  "Dynamic CCE predicate"
  (and (not (cce/using-termux))
       (not (string-blank-p (getenv "DISPLAY")))))

(defun cce/using-linux ()
  "Dynamic CCE predicate"
  (eq 'gnu/linux system-type))

CCE Git Ignore

*.el
*.elc
*.shell
*.conf

*-capture.org
*-template.org
morning-routine-capture
reference-capture-template
gnus-respond
gnus-review
remote-video-capture-template

*.desktop
*.service

emacs-pager-bin
mbsync-endpointrc

termux-url-opener
termux-imap-tunnel

inventory
*.yml
roles

ggs

bash_profile.d
bashrc.d

syncthing.list

userChrome.css
userContent.css

Xmodmap
udev/*

This is Referenced

''

in "CCE Dynamic Loader"

I have the code in Shared CCE Helpers which will generate an init.el from these. To compile this, for now, though elisp:(cce/generate-dynamic-init (cce/module-list)) can be evaluated.

in "CCE on Termux"

in "The Complete Computing Environment"

in "The Complete Computing Environment"

in "Generate a Dynamic Home Manager Configuration"

in "Generate a Dynamic Home Manager Configuration"