User:Tony Sidaway/scripts/search

From Wikipedia, the free encyclopedia

Iron Chicken script:

(define (page-contains-pattern wiki page pattern)
  (and-let* (
    [page-contents (wiki-page-contents wiki page)])
    (string-search pattern page-contents)))

(define (wiki-section-every n lst . opt)
  (let ([label (if (null? opt)
                 (string-append "Section of " (->string n) " number")
                 (car opt))])
    (let loop ([i 0] [lst lst] [out-lst '()])
      (cond
        [(null? lst) out-lst]
        [(< (length lst) n)
          (append
            out-lst
            (list (wiki-heading 3 (string-append label " " (->string i))))
            lst)]
        [else
          (loop
            (+ i 1) 
            (drop lst n)
            (append
              out-lst
              (list (wiki-heading 3 (string-append label " " (->string i))))
              (take lst n)))]))))


(define (delete-adjacent-duplicates lst . opt)
  (let ((eq-pred (if (null? opt) equal? (car opt))))
    (fold-right
      (lambda (elem ret)
        (if (eq-pred elem (first ret))
        ret
        (cons elem ret)))
      (list (last lst))
      lst)))

(define (pages-containing-string wiki string exclusion-list)
  (let* (
    [possible-matches
      (delete-adjacent-duplicates
        (sort
          (filter
            (lambda (page) (not (member page exclusion-list)))
            (search (string-append "\"" string "\"") 50000))
          string<?)
        string=?)]
    [matches
      (filter
        (lambda (page)
          (page-contains-pattern wiki page
            (make-pattern string)))
        possible-matches)])
    matches))

(define (wiki-page->exclusion-list wiki-content)
  (and wiki-content
    (map wiki-unlink
      (wiki-list->list
        (wiki-remove-list-preamble
          (string-split
            (wiki-clean wiki-content) "\n" #t))))))

(let* (
       [search-string
          (with-input-from-string (second (argv)) read-string)]
       [output-page
          (if (> (length (argv)) 2)
            (wiki-unlink (with-input-from-string (third (argv)) read-string))
            (string-append "User:" client-username "/searches/" search-string))]
       [exclusion-list-page
          (and (> (length (argv)) 3)
             (wiki-unlink
               (with-input-from-string (fourth (argv)) read-string)))]
       [exclusion-list (if exclusion-list-page (wiki-page->exclusion-list (wiki-page-contents wiki exclusion-list-page)) '())]
       [result (pages-containing-string wiki search-string exclusion-list)]
       [link-function
         (if (> (length result) 1000)
         wiki-link
         (lambda (x) (string-append "{{la|" x "}}")))])
  (write-wiki-page
    output-page
    (string-append
       (wiki-heading 2 (string-append "Pages unexpectedly containing the phrase \"" search-string "\""))
       (big-apply string-append
         (wiki-section-every 100
           (wiki-list (map link-function result)))))
    (string-append
      "List of pages unexpectedly containing the phrase \""
      search-string "\" "
      "compiled for " client-username " by " bot-username " using " (irnc-user-agent))))