From 581d00ac7cde6852953799edf332b51c7ba62c6e Mon Sep 17 00:00:00 2001
From: Valentin Boettcher <valentin@boettcher.cf>
Date: Wed, 14 Aug 2019 11:23:56 +0200
Subject: [PATCH] add Docs

---
 README.md        | 19 +++++++++++++++++++
 generate-api.ros |  7 ++++---
 scrape.lisp      | 17 +++++++++--------
 3 files changed, 32 insertions(+), 11 deletions(-)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..a83658d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,19 @@
+# CL-SCRAPE-TELEGRAM-API
+
+Scrapes the online [api reference](https://core.telegram.org/bots/api)
+for methods an type definitions.  The scraped methods and types are
+transformed into lisp `functions` and `classes` to be used in the
+`cl-telegram` package.
+
+## Usage
+ 1. Use the command line utility `generate-api.ros [output-file]
+[api-url]` with both args optional.
+ 2. Load the system in the repl and execute `(tg-scrape:scrape-to-disk)`.
+    ```lisp
+    (ql:quickload :cl-scrape-telegram-api)
+    (tg-scrape:scrape-to-disk :out-file "file.lisp")
+    ```
+
+## Todo
+ - Generate type checking for arrays
+ - Generate constructors
diff --git a/generate-api.ros b/generate-api.ros
index 02497bf..0d8d56c 100755
--- a/generate-api.ros
+++ b/generate-api.ros
@@ -4,8 +4,9 @@
 exec ros -Q -- $0 "$@"
 |#
 
-(load "./utils.lisp")
-(load "./scrape.lisp")
+(push #p"./" asdf:*central-registry*)
+(ql:quickload "cl-scrape-telegram-api")
 (in-package :space.protagon.cl-telegram-scrape)
-(defun main (out-file &optional (api-url *url*))
+(defun main (&optional (out-file *out-file*) (api-url *url*))
+  (format t "Writing generated file to ~a.~%" out-file)
   (scrape-to-disk :out-file out-file :url api-url))
diff --git a/scrape.lisp b/scrape.lisp
index 9fc2087..a71da48 100644
--- a/scrape.lisp
+++ b/scrape.lisp
@@ -8,20 +8,19 @@
 (defvar *out-file* "out/out.lisp")
 
 ;; Unimportant categories
-(defconstant +unimportant-categories+ #("Recent Changes"
+(defparameter *unimportant-categories* #("Recent Changes"
                                        "Authorizing your bot"
                                        "Making requests"
                                        "Getting updates"
                                        "Payments"))
 
-(defconstant +type-map+
+(defparameter *type-map*
   '(("Integer" . integer)
     ("String" . string)
     ("Boolean" . boolean)))
 
-(defconstant +blacklist+
-  #("Formatting options" "Inline mode methods" "CallbackGame")
-  "Headers not to be mistaken for methods or types.")
+(defparameter *blacklist*
+    #("Formatting options" "Inline mode methods" "CallbackGame"))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
                                         ;              Structures             ;
@@ -55,7 +54,7 @@
   (let ((cat-data
           (remove-if
            #'(lambda (el)
-               (find (lquery-funcs:text el) +unimportant-categories+ :test 'string-equal))
+               (find (lquery-funcs:text el) *unimportant-categories* :test 'string-equal))
            (lquery:$ *parsed-content* "#dev_page_content h3"))))
     (map 'list #'(lambda (el) (cons (lquery-funcs:text el) el)) cat-data)))
 
@@ -64,7 +63,7 @@
   (let ((types (ppcre:split " or " type-str)))
     (mapcar #'(lambda (single-type)
                 (let* ((is-array (search "Array of " single-type))
-                       (type (assoc single-type +type-map+ :test #'string=)))
+                       (type (assoc single-type *type-map* :test #'string=)))
                   (if is-array nil      ;todo support arrays
                       (if type
                           (cdr type)
@@ -116,7 +115,7 @@
   (let ((name (lquery:$1 h4 (text))))
     (when (or (not (and (lquery:$ h4 (is "h4"))
                    (lquery:$ h4 (next) (is "p"))))
-             (find name +blacklist+ :test #'string=))
+             (find name *blacklist* :test #'string=))
       (return-from parse-h4 nil))
 
     (let* ((anchor (lquery:$1 h4 "a" (attr :href)))
@@ -257,6 +256,8 @@
 
 (defun generate-and-write-functions ()
   "Discovers and generates methods from the telegram api as funcions and writes them to a file."
+  (ensure-directories-exist *out-file*)
+
   (with-open-file (out *out-file* :direction :output :if-exists :supersede)
     (write-file-header out)
     (-> (find-categories) (parse-categories) (print-objects out))))