IN ADDITION to Drupal pages and almost 26,000 WordPress blog posts (and videos and podcasts, not to mention loads of IRC logs) we also have nearly 700 Wiki pages and a reader decided to organise an archive or an index for an archive.
#< fig code > # comments | built-in functions | user functions | inline python
#### license: creative commons cc0 1.0 (public domain) #### http://creativecommons.org/publicdomain/zero/1.0/ proginf = "wikidump 0.1 jul 2019 mn" # dump xml for each wiki page on techrights (no edit hist) # create each actual file function dumppage url urltext = arrcurl url title = "" forin each urltext findtitle = instr each "<title>" iftrue = findtitle title = each ltrim rtrim split title "<title>" join title "" split title "</title>" join title "" break fig next iftrue title outfilename = "techrights_" plus title plus ".xml" split outfilename "/" join outfilename ":slash:" open "w" forin each urltext now = each fprint outfilename next now = outfilename close fig fig
# get list of pages allpages = arrcurl "http://techrights.org/wiki/index.php/Special:AllPages" allpageslen = allpages len longest = 0 longestindex = 0 for each 1 allpageslen 1 eachlen = allpages mid each 1 len ifmore eachlen longest longest = eachlen longestindex = each fig next
# process list of pages and call dumppage for each quot = 34 chr pages = allpages mid longestindex 1 split pages quot forin each pages iswiki = instr each "/wiki/index.php/" ifequal each "/wiki/index.php/Special:AllPages" ignoreit else iftrue iswiki now = "http://techrights.org" plus each split now "http://techrights.org/wiki/index.php/" join now "http://techrights.org/wiki/index.php/Special:Export/" dumppage now fig fig next
# create tgz archive pos = 0 python if figosname != "nt": pos = 1 fig iftrue pos tm = time split tm ":" join tm "." dt = date split dt "/" join dt "-" tgzname = "techrightswiki_" plus dt plus "_" plus tm plus ".tar.gz" now = "tar -cvzf " plus tgzname plus " techrights_*" shell fig
#!/usr/bin/env python2 # encoding: utf-8 # fig translator version: fig 4.6 import sys, os from sys import stdin, stdout from sys import argv as figargv try: from colorama import init init() except: pass # (only) windows users want colorama installed or ansi.sys enabled
try: from sys import exit as quit except: pass
from random import randint from time import sleep
from os import chdir as figoch from os import popen as figpo from os import system as figsh from os import name as figosname figsysteme = 0 figfilehandles = {} figfilecounters = {} from sys import stdout def figlocate(x, l = "ignore", c = "ignore"): import sys if l == "ignore" and c == "ignore": pass # do nothing. want it to return an error?
elif l< 1 and c != "ignore": sys.stdout.write("[" + str(c) + "G") # not ansi.sys compatible elif l != "ignore" and c == "ignore": sys.stdout.write("[" + str(l) + ";" + str(1) + "H") else: sys.stdout.write("[" + str(l) + ";" + str(c) + "H")
import time
def fignonz(p, n=None): if n==None: if p == 0: return 1 else: if p == 0: return n return p
def fignot(p): if p: return 0 return -1
figbac = None figprsbac = None sub = None def fignone(p, figbac): if p == None: return figbac return p return -1
def stopgraphics(): global yourscreen global figraphics figraphics = 0 try: pygame.quit() except: pass
def figcolortext(x, f): b = 0 if f == None: f = 0 if b == None: b = 0 n = "0" if f > 7: n = "1" f = f - 8
if f == 1: f = 4 ## switch ansi colours for qb colours elif f == 4: f = 1 ## 1 = blue not red, 4 = red not blue, etc.
if f == 3: f = 6 elif f == 6: f = 3
if b > 7: b = b - 8
if b == 1: b = 4 elif b == 4: b = 1
if b == 3: b = 6 elif b == 6: b = 3
stdout.write("\x1b[" + n + ";" + str(30+f) + "m") return "\x1b[" + n + ";" + str(30+f) + ";" + str(40+b) + "m"
def figcolourtext(x, f): b = 0 if f == None: f = 0 if b == None: b = 0 n = "0" if f > 7: n = "1" f = f - 8
if f == 1: f = 4 ## switch ansi colours for qb colours elif f == 4: f = 1 ## 1 = blue not red, 4 = red not blue, etc.
if f == 3: f = 6 elif f == 6: f = 3
if b > 7: b = b - 8
if b == 1: b = 4 elif b == 4: b = 1
if b == 3: b = 6 elif b == 6: b = 3
stdout.write("\x1b[" + n + ";" + str(30+f) + "m") return "\x1b[" + n + ";" + str(30+f) + ";" + str(40+b) + "m"
figcgapal = [(0, 0, 0), (0, 0, 170), (0, 170, 0), (0, 170, 170), (170, 0, 0), (170, 0, 170), (170, 85, 0), (170, 170, 170), (85, 85, 85), (85, 85, 255), (85, 255, 85), (85, 255, 255), (255, 85, 85), (255, 85, 255), (255, 255, 85), (255, 255, 255)]
def figget(p, s): return s
def fighighlight(x, b): f = None if f == None: f = 0 if b == None: b = 0 n = "0" if f > 7: n = "1" f = f - 8
if f == 1: f = 4 ## switch ansi colours for qb colours elif f == 4: f = 1 ## 1 = blue not red, 4 = red not blue, etc.
if f == 3: f = 6 elif f == 6: f = 3
if b > 7: b = b - 8
if b == 1: b = 4 elif b == 4: b = 1
if b == 3: b = 6 elif b == 6: b = 3
stdout.write("\x1b[" + n + str(40+b) + "m") return "\x1b[" + n + str(40+b) + "m"
def figinstr(x, p, e): try: return p.index(e) + 1 except: return 0
def figchdir(p): try: figoch(p) except: print "no such file or directory: " + str(p) figend(1)
def figshell(p): global figsysteme try: figsysteme = figsh(p) except: print "error running shell command: " + chr(34) + str(p) + chr(34) figend(1)
def figarrshell(c): global figsysteme try: figsysteme = 0 sh = figpo(c) ps = sh.read().replace(chr(13) + chr(10), chr(10)).replace(chr(13), chr(10)).split(chr(10)) figsysteme = sh.close() except: print "error running arrshell command: " + chr(34) + str(c) + chr(34) figend(1) return ps[:]
def figsgn(p): p = float(p) if p > 0: return 1 if p< 0: return -1 return 0
def figstr(p): return str(p)
def figprint(p): print p
def figchr(p): if type(p) == str: if len(p) > 0: return p[0] return chr(p)
def figprints(p): stdout.write(str(p)) sys.stdout.flush()
def figleft(p, s): return p[:s]
def figmid(p, s, x): arr = 0 if type(p) == list or type(p) == tuple: arr = 1 rt = p[s - 1: x + s - 1] if arr and len(rt) == 1: rt = rt[0] return rt
def figright(p, s): return p[-s:]
def figrandint(x, s, f): return randint(s, f)
def figlcase(p): return p.lower()
def figucase(p): return p.upper()
def figint(p): return int(p)
def figarrset(x, p, s): if type(s) == tuple: if len(s) == 1: fas = s[0] elif type(s) == list: if len(s) == 1: fas = s[0] else: fas = s x[p - 1] = s
def figopen(x, s): import fileinput if s.lower() == "w": if (x) not in figfilehandles.keys(): figfilehandles[x] = open(x[:], s.lower()) elif s.lower() == "r": if (x) not in figfilehandles.keys(): figfilehandles[x] = fileinput.input(x[:]) figfilecounters[x] = 0 else: if (x) not in figfilehandles.keys(): figfilehandles[x] = open(x[:], s[:])
def figfprint(x, s): fon = figosname sep = chr(10) if fon == "nt": sep = chr(13) + chr(10) figfilehandles[s].write(str(x) + sep)
def figflineinput(x, s): try: p = figfilehandles[s][figfilecounters[s]].replace(chr(13), "").replace(chr(10), "") figfilecounters[s] += 1 except: p = chr(10) return p
def figclose(x): if (x) in figfilehandles.keys(): figfilehandles[x].close() del figfilehandles[x] try: del figfilecounters[x] except: pass
def figcls(x): if figosname == "nt": cls = figsh("cls") else: stdout.write("[2J[1;1H") sys.stdout.flush()
def figarropen(x, s): x = open(s).read().replace(chr(13) + chr(10), chr(10)).replace(chr(13), chr(10)).split(chr(10)) return x[:]
def figarrcurl(x, s): from urllib import urlopen x = str(urlopen(s).read()) x = x.replace(chr(13) + chr(10), chr(10)).replace(chr(13), chr(10)).split(chr(10)) return x[:]
def figarrstdin(x): ps = [] for p in stdin: ps += [p[:-1]] return ps[:]
def figarrget(x, p, s): if 1: return p[s - 1]
def figplus(p, s): if type(p) in (float, int): if type(s) in (float, int): p = p + s else: p = p + s # float(s) if you want it easier if p == float(int(p)): p = int(p) else: if type(p) == str: p = p + s # str(s) if you want it easier if type(p) == list: if type(s) == tuple: p = p + list(s) elif type(s) == list: p = p + s[:] else: p = p + [s] if type(p) == tuple: if type(s) == tuple: p = tuple(list(p) + list(s)) elif type(s) == list: p = tuple(list(p) + s[:]) else: p = tuple(list(p) + [s]) return p
def figjoin(p, x, s): t = "" if len(x): t = str(x[0]) for c in range(len(x)): if c > 0: t += str(s) + str(x[c]) return t # s.join(x)
def figarr(p): if type(p) in (float, int, str): p = [p] else: p = list(p) return p
def figsplit(p, x, s): return x.split(s)
def figval(n): n = float(n) if float(int(n)) == float(n): n = int(n) return n
def figtimes(p, s): if type(p) in (float, int): p = p * s # float(s) if you want it easier if p == float(int(p)): p = int(p) else: if type(p) == list: p = p[:] * s # figval(s) else: p = p * s # figval(s) if you want it easer return p
def figdivby(p, s): p = float(p) / s if p == float(int(p)): p = int(p) return p def figminus(p, s): return p - s
def figtopwr(p, s): p = p ** s if p == float(int(p)): p = int(p) return p def figmod(p, s): return p % s
def figcos(p): from math import cos p = cos(p) if p == float(int(p)): p = int(p) return p
def figsin(p): from math import sin p = sin(p) if p == float(int(p)): p = int(p) return p def figsqr(p): from math import sqrt p = sqrt(p) if p == float(int(p)): p = int(p) return p
def figltrim(p): return p.lstrip() def figlineinput(p): return raw_input() def figlen(p): return len(p) def figasc(p): return ord(p[0]) def figatn(p): from math import atan p = atan(p) if p == float(int(p)): p = int(p) return p
def fighex(p): return hex(p) def figrtrim(p): return p.rstrip() def figstring(x, p, n): if type(n) == str: return n * p return chr(n) * p def figtimer(p): from time import strftime return int(strftime("%H"))*60*60+int(strftime("%M"))*60+int(strftime("%S"))
def figtime(p): from time import strftime return strftime("%H:%M:%S")
def figdate(p): from time import strftime return strftime("%m/%d/%Y")
def figcommand(p): return figargv[1:]
def figtan(p): from math import tan p = tan(p) if p == float(int(p)): p = int(p) return p
def figoct(p): return oct(p)
def figsleep(p, s): sleep(s) def figarrsort(p): p.sort()
def figdisplay(x): global figraphics, figrupd figrupd = 0 if figraphics == 1: pygame.display.update()
def figreverse(p): if type(p) == list: p.reverse() return p elif type(p) == str: p = map(str, p) p.reverse() p = "".join(p) return p
def figarreverse(p): p.reverse()
def figfunction(p, s): return p
def figend(x): quit()
def figif(p, s): return p
def figthen(p, s): return p
def figsystem(x): quit()
#### http://creativecommons.org/publicdomain/zero/1.0/
figlist = 0 try: figlist = int(type(proginf) == list) except NameError: pass if not figlist: proginf = 0 proginf = "wikidump 0.1 jul 2019 mn"
# dump xml for each wiki page on techrights (no edit hist)
# create each actual file def dumppage(url):
figlist = 0 try: figlist = int(type(urltext) == list) except NameError: pass if not figlist: urltext = 0 urltext = figarrcurl(urltext, url)
figlist = 0 try: figlist = int(type(title) == list) except NameError: pass if not figlist: title = 0 title = ""
for each in urltext:
figlist = 0 try: figlist = int(type(findtitle) == list) except NameError: pass if not figlist: findtitle = 0 findtitle = figinstr(findtitle, each, "<title>")
if findtitle:
figlist = 0 try: figlist = int(type(title) == list) except NameError: pass if not figlist: title = 0 title = each title = figltrim(title) title = figrtrim(title) title = figsplit(title, title, "<title>") title = figjoin(title, title, "") title = figsplit(title, title, "</title>") title = figjoin(title, title, "")
break if title:
figlist = 0 try: figlist = int(type(outfilename) == list) except NameError: pass if not figlist: outfilename = 0 outfilename = "techrights_" outfilename = figplus(outfilename, title) outfilename = figplus(outfilename, ".xml") outfilename = figsplit(outfilename, outfilename, "/") outfilename = figjoin(outfilename, outfilename, ":slash:") figopen(outfilename, "w")
for each in urltext:
figlist = 0 try: figlist = int(type(now) == list) except NameError: pass if not figlist: now = 0 now = each figfprint(now, outfilename)
figlist = 0 try: figlist = int(type(now) == list) except NameError: pass if not figlist: now = 0 now = outfilename figclose(now)
# get list of pages
figlist = 0 try: figlist = int(type(allpages) == list) except NameError: pass if not figlist: allpages = 0 allpages = figarrcurl(allpages, "http://techrights.org/wiki/index.php/Special:AllPages")
figlist = 0 try: figlist = int(type(allpageslen) == list) except NameError: pass if not figlist: allpageslen = 0 allpageslen = allpages allpageslen = figlen(allpageslen)
figlist = 0 try: figlist = int(type(longest) == list) except NameError: pass if not figlist: longest = 0 longest = 0
figlist = 0 try: figlist = int(type(longestindex) == list) except NameError: pass if not figlist: longestindex = 0 longestindex = 0
for each in range(int(float(1)), int(float(allpageslen)) + figsgn(1), fignonz(int(float(1)))):
figlist = 0 try: figlist = int(type(eachlen) == list) except NameError: pass if not figlist: eachlen = 0 eachlen = allpages eachlen = figmid(eachlen, each, 1) eachlen = figlen(eachlen)
if eachlen > longest:
figlist = 0 try: figlist = int(type(longest) == list) except NameError: pass if not figlist: longest = 0 longest = eachlen
figlist = 0 try: figlist = int(type(longestindex) == list) except NameError: pass if not figlist: longestindex = 0 longestindex = each
# process list of pages and call dumppage for each
figlist = 0 try: figlist = int(type(quot) == list) except NameError: pass if not figlist: quot = 0 quot = 34 quot = figchr(quot)
figlist = 0 try: figlist = int(type(pages) == list) except NameError: pass if not figlist: pages = 0 pages = allpages pages = figmid(pages, longestindex, 1) pages = figsplit(pages, pages, quot)
for each in pages:
figlist = 0 try: figlist = int(type(iswiki) == list) except NameError: pass if not figlist: iswiki = 0 iswiki = figinstr(iswiki, each, "/wiki/index.php/")
if each == "/wiki/index.php/Special:AllPages":
figlist = 0 try: figlist = int(type(ignoreit) == list) except NameError: pass if not figlist: ignoreit = 0
else: if iswiki:
figlist = 0 try: figlist = int(type(now) == list) except NameError: pass if not figlist: now = 0 now = "http://techrights.org" now = figplus(now, each) now = figsplit(now, now, "http://techrights.org/wiki/index.php/") now = figjoin(now, now, "http://techrights.org/wiki/index.php/Special:Export/") figbac = now now = dumppage(now) ; now = fignone(now, figbac) ;
# create tgz archive
figlist = 0 try: figlist = int(type(pos) == list) except NameError: pass if not figlist: pos = 0 pos = 0
if figosname != "nt": pos = 1 if pos:
figlist = 0 try: figlist = int(type(tm) == list) except NameError: pass if not figlist: tm = 0 tm = figtime(tm) tm = figsplit(tm, tm, ":") tm = figjoin(tm, tm, ".")
figlist = 0 try: figlist = int(type(dt) == list) except NameError: pass if not figlist: dt = 0 dt = figdate(dt) dt = figsplit(dt, dt, "/") dt = figjoin(dt, dt, "-")
figlist = 0 try: figlist = int(type(tgzname) == list) except NameError: pass if not figlist: tgzname = 0 tgzname = "techrightswiki_" tgzname = figplus(tgzname, dt) tgzname = figplus(tgzname, "_") tgzname = figplus(tgzname, tm) tgzname = figplus(tgzname, ".tar.gz")
figlist = 0 try: figlist = int(type(now) == list) except NameError: pass if not figlist: now = 0 now = "tar -cvzf " now = figplus(now, tgzname) now = figplus(now, " techrights_*") figshell(now)