diff --git a/addons/wiki/.project b/addons/wiki/.project new file mode 100755 index 00000000000..c42ef35701d --- /dev/null +++ b/addons/wiki/.project @@ -0,0 +1,11 @@ + + + wiki + + + + + + + + diff --git a/addons/wiki/Cell.py b/addons/wiki/Cell.py new file mode 100755 index 00000000000..de2c606eb8c --- /dev/null +++ b/addons/wiki/Cell.py @@ -0,0 +1,18 @@ +# Copyright (C) 2006 Samuel Abels, http://debain.org +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +class Cell: + data = '' + colspan = 1 + rowspan = 1 diff --git a/addons/wiki/Html2Wiki.py b/addons/wiki/Html2Wiki.py new file mode 100755 index 00000000000..e16ad08fd19 --- /dev/null +++ b/addons/wiki/Html2Wiki.py @@ -0,0 +1,290 @@ +# Copyright (C) 2006 Samuel Abels, http://debain.org +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import HTMLParser, re, sys +from Cell import Cell + +class Html2Wiki(HTMLParser.HTMLParser): + def __init__(self): + HTMLParser.HTMLParser.__init__(self) + self.wiki = '' + self.buffer = '' + self.indent = 0 + self.linebreak = '\n' + self.rows = [] + self.cells = [] + self.in_table = False + self.in_td = False + self.in_heading = False + self.in_ul = False + self.in_ol = False + self.in_li = False + self.in_a = False + self.in_pre = False + self.last_href = '' + + def __output(self, text, linebreak = True): + self.buffer += (' ' * self.indent * 2) + self.buffer += text + self.linebreak + + def __flush(self): + self.wiki += self.buffer + self.buffer = '' + + def handle_starttag(self, tag, attrs): + if tag == 'table': self.start_table() + elif tag == 'tr': self.start_tr() + elif tag == 'th': self.start_th(attrs) + elif tag == 'td': self.start_td(attrs) + elif tag == 'h1': self.start_h1() + elif tag == 'h2': self.start_h2() + elif tag == 'h3': self.start_h3() + elif tag == 'ul': self.start_ul() + elif tag == 'ol': self.start_ol() + elif tag == 'li': self.start_li() + elif tag == 'i': self.start_i() + elif tag == 'b': self.start_b() + elif tag == 'u': self.start_u() + elif tag == 'a': self.start_a(attrs) + elif tag == 'pre': self.start_pre() + elif tag == 'strike': self.start_strike() + elif tag == 'br': self.newline() + + def handle_endtag(self, tag): + if tag == 'table': self.end_table(); + elif tag == 'tr': self.end_tr() + elif tag == 'th': self.end_th() + elif tag == 'td': self.end_td() + elif tag == 'h1': self.end_h1() + elif tag == 'h2': self.end_h2() + elif tag == 'h3': self.end_h3() + elif tag == 'ul': self.end_ul() + elif tag == 'ol': self.end_ol() + elif tag == 'li': self.end_li() + elif tag == 'i': self.end_i() + elif tag == 'b': self.end_b() + elif tag == 'u': self.end_u() + elif tag == 'a': self.end_a() + elif tag == 'pre': self.end_pre() + elif tag == 'strike': self.end_strike() + + def start_h1(self): + self.buffer += '=' + + def end_h1(self): + self.buffer += '=\n\n' + + def start_h2(self): + self.buffer += '==' + + def end_h2(self): + self.buffer += '==\n\n' + + def start_h3(self): + self.buffer += '===' + + def end_h3(self): + self.buffer += '===\n\n' + + def start_ul(self): + self.in_ul = True + + def end_ul(self): + self.in_ul = False + + def start_ol(self): + self.in_ol = True + + def end_ol(self): + self.in_ol = False + + def start_li(self): + self.in_li = True + if self.in_ol: + self.buffer += '# ' + elif self.in_ul: + self.buffer += '* ' + + def end_li(self): + self.in_li = False + + def start_i(self): + self.buffer += '/' + + def end_i(self): + self.buffer += '/' + + def start_b(self): + self.buffer += '*' + + def end_b(self): + self.buffer += '*' + + def start_u(self): + self.buffer += '_' + + def end_u(self): + self.buffer += '_' + + def start_a(self, attrs): + self.in_a = True + self.last_href = '' + for key, value in attrs: + if key == 'href': + self.last_href = value + self.buffer += '[' + self.last_href + + def end_a(self): + self.in_a = False + self.buffer += ']' + + def start_pre(self): + self.in_pre = True + self.buffer += '#Text\n' + + def end_pre(self): + self.in_pre = False + self.buffer += '#End\n' + + def start_strike(self): + self.buffer += '-' + + def end_strike(self): + self.buffer += '-' + + def start_table(self): + self.in_table = True + + def start_tr(self): + pass + + def start_th(self, attrs): + self.in_heading = True + self.start_td(attrs) + + def start_td(self, attrs): + self.__flush() + self.in_td = True + cell = Cell() + for key, value in attrs: + if key == 'rowspan': + cell.rowspan = int(value) + elif key == 'colspan': + cell.colspan = int(value) + self.cells.append(cell) + + def handle_data(self, data): + if not self.in_pre: + data = data.replace('\n', '') + if self.in_a: + if data == self.last_href: + return + self.buffer += ' ' + if self.in_li: + self.buffer += data.strip() + '\n' + if self.in_ul or self.in_ol: + self.__flush() + elif self.in_td: + self.buffer += data + elif not self.in_table: + self.buffer += data + self.__flush() + + def end_td(self): + self.cells[-1].data += self.buffer.strip() + self.buffer = '' + self.in_td = False + + def end_th(self): + self.end_td() + + def end_tr(self): + if len(self.cells) is 0: + return + if self.in_heading: + self.__output('#Heading') + self.in_heading = False + else: + self.__output('#Row') + self.indent += 1 + line = ('|' * self.cells[0].colspan) + ' ' + self.cells[0].data.strip() + for cell in self.cells[1:]: + line += ' ' + ('|' * cell.colspan) + ' ' + cell.data.strip() + if len(line) <= 80: + self.__output(line) + else: + for cell in self.cells: + self.__output(('|' * cell.colspan) + ' ' + cell.data.strip()) + self.cells = [] + self.indent -= 1 + self.__flush() + + def end_table(self): + self.in_table = False + self.__flush() + + def newline(self): + self.buffer += '\n' + + +if __name__ == '__main__': + import unittest + import os + from Wiki2Html import Wiki2Html + + class Wiki2HtmlTest(unittest.TestCase): + def runTest(self): + # Read the entire file into one string. + filename = 'markup.txt' + infile = open(filename, 'r') + in_str = infile.read() + infile.close() + + # Convert Wiki to HTML. + parser = Wiki2Html() + parser.read(filename) + html1 = parser.html + #print html1 + + # Convert the HTML back to Wiki. + parser = Html2Wiki() + parser.feed(html1) + wiki = parser.wiki + #print wiki + + # Write the wiki to a file. + # The result contains an extra newline to mark a table end. + # This is generally ok, but would break the unit test, so we + # remove the last character before writing. + fd = open(filename + '.tmp', 'w') + fd.write(wiki[:-1]) + fd.close() + + # Convert the new Wiki file to HTML again. + parser = Wiki2Html() + parser.read(filename + '.tmp') + html2 = parser.html + #print html2 + + # Clean up. + os.remove(filename + '.tmp') + + # Make sure that the model is complete. + assert len(in_str) > 10 + assert len(html1) > 10 + assert html1 == html2 + + testcase = Wiki2HtmlTest() + runner = unittest.TextTestRunner() + runner.run(testcase) diff --git a/addons/wiki/IndexClient.py b/addons/wiki/IndexClient.py new file mode 100755 index 00000000000..1bf036f9e20 --- /dev/null +++ b/addons/wiki/IndexClient.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +import xmlrpclib + +tiny_host = 'localhost' +tiny_port = 8000 + +url = 'http://%s:%s' % (tiny_host, tiny_port) +proxy = xmlrpclib.ServerProxy(url); + +proxy.init('terp', 3, 'admin') +print proxy.search('china') \ No newline at end of file diff --git a/addons/wiki/IndexServer.py b/addons/wiki/IndexServer.py new file mode 100755 index 00000000000..60adcd17bba --- /dev/null +++ b/addons/wiki/IndexServer.py @@ -0,0 +1,178 @@ +#!/usr/bin/python + +from SimpleXMLRPCServer import SimpleXMLRPCServer +import os +import csv +from config import * +import xmlrpclib +import PyLucene +import threading + +class IndexServer(threading.Thread): + + def __init__(self): + super(IndexServer,self).__init__() + self.path = os.path.join('/tmp', 'index') + + def initIndex(self, database, user, password, new=False): + self.database = database + self.user = user + self.password = password + self.new = new + self.writer = None + + def open(self): + if not os.path.exists(self.path): + os.mkdir(path) + self.new = True + + store = PyLucene.FSDirectory.getDirectory(path, self.new) + self.writer = PyLucene.IndexWriter(store, PyLucene.StandardAnalyzer(), self.new) + url = 'http://%s:%s/xmlrpc/object' % (tiny_host, str(tiny_port)) + self.proxy = xmlrpclib.ServerProxy(url); + return True + + def addDocument(self, model, id, data): + + doc = PyLucene.Document() + doc.add(PyLucene.Field("model", model, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.UN_TOKENIZED)) + indexid=str(id) + + doc.add(PyLucene.Field("id", indexid, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.UN_TOKENIZED)) + + doc.add(PyLucene.Field("contents", data, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.TOKENIZED)) + self.writer.addDocument(doc) + + return True + + def doIndex(self, models=None): + + self.open() + + toIndex = [] + if models is None: + toIndex = self.proxy.execute(self.database, self.user, self.password, 'ir.model' ,'search', []) + else: + toIndex = models + + models = self.proxy.execute(self.database, self.user, self.password, 'ir.model' ,'read', toIndex, ['model']) + for mod in models: + res_ids = self.proxy.execute(self.database, self.user, self.password, mod['model'], 'search', []) + + if str(mod['model']).startswith('ir'): + continue + + if str(mod['model']).startswith('res.currency'): + continue + + print 'creating index for : ', str(mod['model']) + try: + res_datas = self.proxy.execute(self.database, self.user, self.password, mod['model'], 'read', res_ids) + except Exception, e: + print e + continue + + for res_data in res_datas: + self.addDocument(mod['model'], res_data['id'], ' '.join(map(str,res_data.values()))) + + self.new = False + self.close() + return True + + def reIndex(self, models = None): + self.new = True + self.doIndex(models) + + def close(self): + self.writer.close() + return True + + def addRecord(self, model, id, data): + self.new = False + self.open() + doc = PyLucene.Document() + doc.add(PyLucene.Field("model", model, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.UN_TOKENIZED)) + indexid=str(id) + + doc.add(PyLucene.Field("id", indexid, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.UN_TOKENIZED)) + + doc.add(PyLucene.Field("contents", data, + PyLucene.Field.Store.YES, + PyLucene.Field.Index.TOKENIZED)) + self.writer.addDocument(doc) + self.close() + return True + + def search(self, keys): + directory = PyLucene.FSDirectory.getDirectory(self.path, False) + searcher = PyLucene.IndexSearcher(directory) + analyzer = PyLucene.StandardAnalyzer() + + query = PyLucene.QueryParser("contents", analyzer).parse(keys) + hits = searcher.search(query) + count= hits.length() + result = {} + + for i, doc in hits: + model = doc.get('model') + id = doc.get('id') + + if result.has_key(model): + ids = result.get(model) + ids.append(id) + else: + ids = [] + ids.append(id) + result[model] = ids + + searcher.close() + + if result.__len__() <=0 : + result = False + + return result + + def run(self): + server = SimpleXMLRPCServer((index_server,index_port),allow_none=True) + indexer = IndexServer() + server.register_instance(indexer, True) + server.register_function(indexer.initIndex, 'init') + server.register_function(indexer.open, 'open') + server.register_function(indexer.addRecord, 'add') + server.register_function(indexer.doIndex, 'index') + server.register_function(indexer.reIndex, 'reindex') + server.register_function(indexer.close, 'close') + server.register_function(indexer.search, 'search') + server.register_introspection_functions() + server.serve_forever() +#end class + +server = SimpleXMLRPCServer((index_server,index_port),allow_none=True) +indexer = IndexServer() +server.register_function(indexer.initIndex, 'init') +server.register_function(indexer.open, 'open') +server.register_function(indexer.addRecord, 'add') +server.register_function(indexer.doIndex, 'index') +server.register_function(indexer.reIndex, 'reindex') +server.register_function(indexer.close, 'close') +server.register_function(indexer.search, 'search') +server.register_introspection_functions() +server.register_multicall_functions() +server.serve_forever() +# +#try: +# print 'Starting Index Server', index_server, index_port +# server = IndexServer() +# server.start() +#except Exception , e: +# print e diff --git a/addons/wiki/Plex/._.DS_Store b/addons/wiki/Plex/._.DS_Store new file mode 100755 index 00000000000..460d887a2e2 Binary files /dev/null and b/addons/wiki/Plex/._.DS_Store differ diff --git a/addons/wiki/Plex/._Actions.py b/addons/wiki/Plex/._Actions.py new file mode 100755 index 00000000000..1ab1d742745 Binary files /dev/null and b/addons/wiki/Plex/._Actions.py differ diff --git a/addons/wiki/Plex/._DFA.py b/addons/wiki/Plex/._DFA.py new file mode 100755 index 00000000000..5fa2642423c Binary files /dev/null and b/addons/wiki/Plex/._DFA.py differ diff --git a/addons/wiki/Plex/._Errors.py b/addons/wiki/Plex/._Errors.py new file mode 100755 index 00000000000..af4b4136e43 Binary files /dev/null and b/addons/wiki/Plex/._Errors.py differ diff --git a/addons/wiki/Plex/._Lexicons.py b/addons/wiki/Plex/._Lexicons.py new file mode 100755 index 00000000000..e133ecc39a1 Binary files /dev/null and b/addons/wiki/Plex/._Lexicons.py differ diff --git a/addons/wiki/Plex/._Machines.py b/addons/wiki/Plex/._Machines.py new file mode 100755 index 00000000000..9abffbaadfa Binary files /dev/null and b/addons/wiki/Plex/._Machines.py differ diff --git a/addons/wiki/Plex/._Regexps.py b/addons/wiki/Plex/._Regexps.py new file mode 100755 index 00000000000..68c7c99266a Binary files /dev/null and b/addons/wiki/Plex/._Regexps.py differ diff --git a/addons/wiki/Plex/._Scanners.py b/addons/wiki/Plex/._Scanners.py new file mode 100755 index 00000000000..42c3ef2ac8d Binary files /dev/null and b/addons/wiki/Plex/._Scanners.py differ diff --git a/addons/wiki/Plex/._Timing.py b/addons/wiki/Plex/._Timing.py new file mode 100755 index 00000000000..a79e35c322d Binary files /dev/null and b/addons/wiki/Plex/._Timing.py differ diff --git a/addons/wiki/Plex/._Traditional.py b/addons/wiki/Plex/._Traditional.py new file mode 100755 index 00000000000..39ffda334b8 Binary files /dev/null and b/addons/wiki/Plex/._Traditional.py differ diff --git a/addons/wiki/Plex/._Transitions.py b/addons/wiki/Plex/._Transitions.py new file mode 100755 index 00000000000..a1afe03ef1d Binary files /dev/null and b/addons/wiki/Plex/._Transitions.py differ diff --git a/addons/wiki/Plex/.___init__.py b/addons/wiki/Plex/.___init__.py new file mode 100755 index 00000000000..88f160bc02e Binary files /dev/null and b/addons/wiki/Plex/.___init__.py differ diff --git a/addons/wiki/Plex/._test_tm.py b/addons/wiki/Plex/._test_tm.py new file mode 100755 index 00000000000..a5fdaefe5e8 Binary files /dev/null and b/addons/wiki/Plex/._test_tm.py differ diff --git a/addons/wiki/Plex/Actions.py b/addons/wiki/Plex/Actions.py new file mode 100755 index 00000000000..23253a905a1 --- /dev/null +++ b/addons/wiki/Plex/Actions.py @@ -0,0 +1,109 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Actions for use in token specifications +# +#======================================================================= + +class Action: + + def same_as(self, other): + return self is other + + +class Return(Action): + """ + Internal Plex action which causes |value| to + be returned as the value of the associated token + """ + + value = None + + def __init__(self, value): + self.value = value + + def perform(self, token_stream, text): + return self.value + + def same_as(self, other): + return isinstance(other, Return) and self.value == other.value + + def __repr__(self): + return "Return(%s)" % repr(self.value) + + +class Call(Action): + """ + Internal Plex action which causes a function to be called. + """ + + function = None + + def __init__(self, function): + self.function = function + + def perform(self, token_stream, text): + return self.function(token_stream, text) + + def __repr__(self): + return "Call(%s)" % self.function.__name__ + + def same_as(self, other): + return isinstance(other, Call) and self.function is other.function + + +class Begin(Action): + """ + Begin(state_name) is a Plex action which causes the Scanner to + enter the state |state_name|. See the docstring of Plex.Lexicon + for more information. + """ + + state_name = None + + def __init__(self, state_name): + self.state_name = state_name + + def perform(self, token_stream, text): + token_stream.begin(self.state_name) + + def __repr__(self): + return "Begin(%s)" % self.state_name + + def same_as(self, other): + return isinstance(other, Begin) and self.state_name == other.state_name + + +class Ignore(Action): + """ + IGNORE is a Plex action which causes its associated token + to be ignored. See the docstring of Plex.Lexicon for more + information. + """ + def perform(self, token_stream, text): + return None + + def __repr__(self): + return "IGNORE" + +IGNORE = Ignore() +IGNORE.__doc__ = Ignore.__doc__ + +class Text(Action): + """ + TEXT is a Plex action which causes the text of a token to + be returned as the value of the token. See the docstring of + Plex.Lexicon for more information. + """ + + def perform(self, token_stream, text): + return text + + def __repr__(self): + return "TEXT" + +TEXT = Text() +TEXT.__doc__ = Text.__doc__ + + diff --git a/addons/wiki/Plex/DFA.py b/addons/wiki/Plex/DFA.py new file mode 100755 index 00000000000..2c0004b092e --- /dev/null +++ b/addons/wiki/Plex/DFA.py @@ -0,0 +1,156 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Converting NFA to DFA +# +#======================================================================= + +import Machines +from Machines import LOWEST_PRIORITY +from Transitions import TransitionMap + +def nfa_to_dfa(old_machine, debug = None): + """ + Given a nondeterministic Machine, return a new equivalent + Machine which is deterministic. + """ + # We build a new machine whose states correspond to sets of states + # in the old machine. Initially we add a new state corresponding to + # the epsilon-closure of each initial old state. Then we give transitions + # to each new state which are the union of all transitions out of any + # of the corresponding old states. The new state reached on a given + # character is the one corresponding to the set of states reachable + # on that character from any of the old states. As new combinations of + # old states are created, new states are added as needed until closure + # is reached. + new_machine = Machines.FastMachine() + state_map = StateMap(new_machine) + # Seed the process using the initial states of the old machine. + # Make the corresponding new states into initial states of the new + # machine with the same names. + for (key, old_state) in old_machine.initial_states.items(): + new_state = state_map.old_to_new(epsilon_closure(old_state)) + new_machine.make_initial_state(key, new_state) + # Tricky bit here: we add things to the end of this list while we're + # iterating over it. The iteration stops when closure is achieved. + for new_state in new_machine.states: + transitions = TransitionMap() + for old_state in state_map.new_to_old(new_state).keys(): + for event, old_target_states in old_state.transitions.items(): + if event and old_target_states: + transitions.add_set(event, set_epsilon_closure(old_target_states)) + for event, old_states in transitions.items(): + new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states)) + if debug: + debug.write("\n===== State Mapping =====\n") + state_map.dump(debug) + return new_machine + +def set_epsilon_closure(state_set): + """ + Given a set of states, return the union of the epsilon + closures of its member states. + """ + result = {} + for state1 in state_set.keys(): + for state2 in epsilon_closure(state1).keys(): + result[state2] = 1 + return result + +def epsilon_closure(state): + """ + Return the set of states reachable from the given state + by epsilon moves. + """ + # Cache the result + result = state.epsilon_closure + if result is None: + result = {} + state.epsilon_closure = result + add_to_epsilon_closure(result, state) + return result + +def add_to_epsilon_closure(state_set, state): + """ + Recursively add to |state_set| states reachable from the given state + by epsilon moves. + """ + if not state_set.get(state, 0): + state_set[state] = 1 + state_set_2 = state.transitions.get_epsilon() + if state_set_2: + for state2 in state_set_2.keys(): + add_to_epsilon_closure(state_set, state2) + +class StateMap: + """ + Helper class used by nfa_to_dfa() to map back and forth between + sets of states from the old machine and states of the new machine. + """ + new_machine = None # Machine + old_to_new_dict = None # {(old_state,...) : new_state} + new_to_old_dict = None # {id(new_state) : old_state_set} + + def __init__(self, new_machine): + self.new_machine = new_machine + self.old_to_new_dict = {} + self.new_to_old_dict= {} + + def old_to_new(self, old_state_set): + """ + Return the state of the new machine corresponding to the + set of old machine states represented by |state_set|. A new + state will be created if necessary. If any of the old states + are accepting states, the new state will be an accepting state + with the highest priority action from the old states. + """ + key = self.make_key(old_state_set) + new_state = self.old_to_new_dict.get(key, None) + if not new_state: + action = self.highest_priority_action(old_state_set) + new_state = self.new_machine.new_state(action) + self.old_to_new_dict[key] = new_state + self.new_to_old_dict[id(new_state)] = old_state_set + #for old_state in old_state_set.keys(): + #new_state.merge_actions(old_state) + return new_state + + def highest_priority_action(self, state_set): + best_action = None + best_priority = LOWEST_PRIORITY + for state in state_set.keys(): + priority = state.action_priority + if priority > best_priority: + best_action = state.action + best_priority = priority + return best_action + +# def old_to_new_set(self, old_state_set): +# """ +# Return the new state corresponding to a set of old states as +# a singleton set. +# """ +# return {self.old_to_new(old_state_set):1} + + def new_to_old(self, new_state): + """Given a new state, return a set of corresponding old states.""" + return self.new_to_old_dict[id(new_state)] + + def make_key(self, state_set): + """ + Convert a set of states into a uniquified + sorted tuple suitable for use as a dictionary key. + """ + lst = state_set.keys() + lst.sort() + return tuple(lst) + + def dump(self, file): + from Transitions import state_set_str + for new_state in self.new_machine.states: + old_state_set = self.new_to_old_dict[id(new_state)] + file.write(" State %s <-- %s\n" % ( + new_state['number'], state_set_str(old_state_set))) + + diff --git a/addons/wiki/Plex/Errors.py b/addons/wiki/Plex/Errors.py new file mode 100755 index 00000000000..ae033672b50 --- /dev/null +++ b/addons/wiki/Plex/Errors.py @@ -0,0 +1,52 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Exception classes +# +#======================================================================= + +import exceptions + +class PlexError(exceptions.Exception): + message = "" + +class PlexTypeError(PlexError, TypeError): + pass + +class PlexValueError(PlexError, ValueError): + pass + +class InvalidRegex(PlexError): + pass + +class InvalidToken(PlexError): + + def __init__(self, token_number, message): + PlexError.__init__(self, "Token number %d: %s" % (token_number, message)) + +class InvalidScanner(PlexError): + pass + +class AmbiguousAction(PlexError): + message = "Two tokens with different actions can match the same string" + + def __init__(self): + pass + +class UnrecognizedInput(PlexError): + scanner = None + position = None + state_name = None + + def __init__(self, scanner, state_name): + self.scanner = scanner + self.position = scanner.position() + self.state_name = state_name + + def __str__(self): + return ("'%s', line %d, char %d: Token not recognised in state %s" + % (self.position + (repr(self.state_name),))) + + + diff --git a/addons/wiki/Plex/Lexicons.py b/addons/wiki/Plex/Lexicons.py new file mode 100755 index 00000000000..32b12c431c3 --- /dev/null +++ b/addons/wiki/Plex/Lexicons.py @@ -0,0 +1,192 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Lexical Analyser Specification +# +#======================================================================= + +import types + +import Actions +import DFA +import Errors +import Machines +import Regexps + +# debug_flags for Lexicon constructor +DUMP_NFA = 1 +DUMP_DFA = 2 + +class State: + """ + This class is used as part of a Plex.Lexicon specification to + introduce a user-defined state. + + Constructor: + + State(name, token_specifications) + """ + + name = None + tokens = None + + def __init__(self, name, tokens): + self.name = name + self.tokens = tokens + +class Lexicon: + """ + Lexicon(specification) builds a lexical analyser from the given + |specification|. The specification consists of a list of + specification items. Each specification item may be either: + + 1) A token definition, which is a tuple: + + (pattern, action) + + The |pattern| is a regular axpression built using the + constructors defined in the Plex module. + + The |action| is the action to be performed when this pattern + is recognised (see below). + + 2) A state definition: + + State(name, tokens) + + where |name| is a character string naming the state, + and |tokens| is a list of token definitions as + above. The meaning and usage of states is described + below. + + Actions + ------- + + The |action| in a token specication may be one of three things: + + 1) A function, which is called as follows: + + function(scanner, text) + + where |scanner| is the relevant Scanner instance, and |text| + is the matched text. If the function returns anything + other than None, that value is returned as the value of the + token. If it returns None, scanning continues as if the IGNORE + action were specified (see below). + + 2) One of the following special actions: + + IGNORE means that the recognised characters will be treated as + white space and ignored. Scanning will continue until + the next non-ignored token is recognised before returning. + + TEXT causes the scanned text itself to be returned as the + value of the token. + + 3) Any other value, which is returned as the value of the token. + + States + ------ + + At any given time, the scanner is in one of a number of states. + Associated with each state is a set of possible tokens. When scanning, + only tokens associated with the current state are recognised. + + There is a default state, whose name is the empty string. Token + definitions which are not inside any State definition belong to + the default state. + + The initial state of the scanner is the default state. The state can + be changed in one of two ways: + + 1) Using Begin(state_name) as the action of a token. + + 2) Calling the begin(state_name) method of the Scanner. + + To change back to the default state, use '' as the state name. + """ + + machine = None # Machine + tables = None # StateTableMachine + + def __init__(self, specifications, debug = None, debug_flags = 7, timings = None): + if type(specifications) <> types.ListType: + raise Errors.InvalidScanner("Scanner definition is not a list") + if timings: + from Timing import time + total_time = 0.0 + time1 = time() + nfa = Machines.Machine() + default_initial_state = nfa.new_initial_state('') + token_number = 1 + for spec in specifications: + if isinstance(spec, State): + user_initial_state = nfa.new_initial_state(spec.name) + for token in spec.tokens: + self.add_token_to_machine( + nfa, user_initial_state, token, token_number) + token_number = token_number + 1 + elif type(spec) == types.TupleType: + self.add_token_to_machine( + nfa, default_initial_state, spec, token_number) + token_number = token_number + 1 + else: + raise Errors.InvalidToken( + token_number, + "Expected a token definition (tuple) or State instance") + if timings: + time2 = time() + total_time = total_time + (time2 - time1) + time3 = time() + if debug and (debug_flags & 1): + debug.write("\n============= NFA ===========\n") + nfa.dump(debug) + dfa = DFA.nfa_to_dfa(nfa, debug = (debug_flags & 3) == 3 and debug) + if timings: + time4 = time() + total_time = total_time + (time4 - time3) + if debug and (debug_flags & 2): + debug.write("\n============= DFA ===========\n") + dfa.dump(debug) + if timings: + timings.write("Constructing NFA : %5.2f\n" % (time2 - time1)) + timings.write("Converting to DFA: %5.2f\n" % (time4 - time3)) + timings.write("TOTAL : %5.2f\n" % total_time) + self.machine = dfa + + def add_token_to_machine(self, machine, initial_state, token_spec, token_number): + try: + (re, action_spec) = self.parse_token_definition(token_spec) + # Disabled this -- matching empty strings can be useful + #if re.nullable: + # raise Errors.InvalidToken( + # token_number, "Pattern can match 0 input symbols") + if isinstance(action_spec, Actions.Action): + action = action_spec + elif callable(action_spec): + action = Actions.Call(action_spec) + else: + action = Actions.Return(action_spec) + final_state = machine.new_state() + re.build_machine(machine, initial_state, final_state, + match_bol = 1, nocase = 0) + final_state.set_action(action, priority = -token_number) + except Errors.PlexError, e: + raise e.__class__("Token number %d: %s" % (token_number, e)) + + def parse_token_definition(self, token_spec): + if type(token_spec) <> types.TupleType: + raise Errors.InvalidToken("Token definition is not a tuple") + if len(token_spec) <> 2: + raise Errors.InvalidToken("Wrong number of items in token definition") + pattern, action = token_spec + if not isinstance(pattern, Regexps.RE): + raise Errors.InvalidToken("Pattern is not an RE instance") + return (pattern, action) + + def get_initial_state(self, name): + return self.machine.get_initial_state(name) + + + diff --git a/addons/wiki/Plex/Machines.py b/addons/wiki/Plex/Machines.py new file mode 100755 index 00000000000..fb9ba717d80 --- /dev/null +++ b/addons/wiki/Plex/Machines.py @@ -0,0 +1,326 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Classes for building NFAs and DFAs +# +#======================================================================= + +import string +import sys +from sys import maxint +from types import TupleType + +from Transitions import TransitionMap + +LOWEST_PRIORITY = -sys.maxint + +class Machine: + """A collection of Nodes representing an NFA or DFA.""" + states = None # [Node] + next_state_number = 1 + initial_states = None # {(name, bol): Node} + + def __init__(self): + self.states = [] + self.initial_states = {} + + def __del__(self): + #print "Destroying", self ### + for state in self.states: + state.destroy() + + def new_state(self): + """Add a new state to the machine and return it.""" + s = Node() + n = self.next_state_number + self.next_state_number = n + 1 + s.number = n + self.states.append(s) + return s + + def new_initial_state(self, name): + state = self.new_state() + self.make_initial_state(name, state) + return state + + def make_initial_state(self, name, state): + self.initial_states[name] = state + + def get_initial_state(self, name): + return self.initial_states[name] + + def dump(self, file): + file.write("Plex.Machine:\n") + if self.initial_states is not None: + file.write(" Initial states:\n") + for (name, state) in self.initial_states.items(): + file.write(" '%s': %d\n" % (name, state.number)) + for s in self.states: + s.dump(file) + +class Node: + """A state of an NFA or DFA.""" + transitions = None # TransitionMap + action = None # Action + action_priority = None # integer + number = 0 # for debug output + epsilon_closure = None # used by nfa_to_dfa() + + def __init__(self): + # Preinitialise the list of empty transitions, because + # the nfa-to-dfa algorithm needs it + #self.transitions = {'':[]} + self.transitions = TransitionMap() + self.action_priority = LOWEST_PRIORITY + + def destroy(self): + #print "Destroying", self ### + self.transitions = None + self.action = None + self.epsilon_closure = None + + def add_transition(self, event, new_state): + self.transitions.add(event, new_state) + + def link_to(self, state): + """Add an epsilon-move from this state to another state.""" + self.add_transition('', state) + + def set_action(self, action, priority): + """Make this an accepting state with the given action. If + there is already an action, choose the action with highest + priority.""" + if priority > self.action_priority: + self.action = action + self.action_priority = priority + + def get_action(self): + return self.action + + def get_action_priority(self): + return self.action_priority + +# def merge_actions(self, other_state): +# """Merge actions of other state into this state according +# to their priorities.""" +# action = other_state.get_action() +# priority = other_state.get_action_priority() +# self.set_action(action, priority) + + def is_accepting(self): + return self.action is not None + + def __str__(self): + return "State %d" % self.number + + def dump(self, file): + import string + # Header + file.write(" State %d:\n" % self.number) + # Transitions +# self.dump_transitions(file) + self.transitions.dump(file) + # Action + action = self.action + priority = self.action_priority + if action is not None: + file.write(" %s [priority %d]\n" % (action, priority)) + + +class FastMachine: + """ + FastMachine is a deterministic machine represented in a way that + allows fast scanning. + """ + initial_states = None # {state_name:state} + states = None # [state] + # where state = {event:state, 'else':state, 'action':Action} + next_number = 1 # for debugging + + new_state_template = { + '':None, 'bol':None, 'eol':None, 'eof':None, 'else':None + } + + def __init__(self, old_machine = None): + self.initial_states = initial_states = {} + self.states = [] + if old_machine: + self.old_to_new = old_to_new = {} + for old_state in old_machine.states: + new_state = self.new_state() + old_to_new[old_state] = new_state + for name, old_state in old_machine.initial_states.items(): + initial_states[name] = old_to_new[old_state] + for old_state in old_machine.states: + new_state = old_to_new[old_state] + for event, old_state_set in old_state.transitions.items(): + if old_state_set: + new_state[event] = old_to_new[old_state_set.keys()[0]] + else: + new_state[event] = None + new_state['action'] = old_state.action + + def __del__(self): + for state in self.states: + state.clear() + + def new_state(self, action = None): + number = self.next_number + self.next_number = number + 1 + result = self.new_state_template.copy() + result['number'] = number + result['action'] = action + self.states.append(result) + return result + + def make_initial_state(self, name, state): + self.initial_states[name] = state + + def add_transitions(self, state, event, new_state): + if type(event) == TupleType: + code0, code1 = event + if code0 == -maxint: + state['else'] = new_state + elif code1 <> maxint: + while code0 < code1: + state[chr(code0)] = new_state + code0 = code0 + 1 + else: + state[event] = new_state + + def get_initial_state(self, name): + return self.initial_states[name] + + def dump(self, file): + file.write("Plex.FastMachine:\n") + file.write(" Initial states:\n") + for name, state in self.initial_states.items(): + file.write(" %s: %s\n" % (repr(name), state['number'])) + for state in self.states: + self.dump_state(state, file) + + def dump_state(self, state, file): + import string + # Header + file.write(" State %d:\n" % state['number']) + # Transitions + self.dump_transitions(state, file) + # Action + action = state['action'] + if action is not None: + file.write(" %s\n" % action) + + def dump_transitions(self, state, file): + chars_leading_to_state = {} + special_to_state = {} + for (c, s) in state.items(): + if len(c) == 1: + chars = chars_leading_to_state.get(id(s), None) + if chars is None: + chars = [] + chars_leading_to_state[id(s)] = chars + chars.append(c) + elif len(c) <= 4: + special_to_state[c] = s + ranges_to_state = {} + for state in self.states: + char_list = chars_leading_to_state.get(id(state), None) + if char_list: + ranges = self.chars_to_ranges(char_list) + ranges_to_state[ranges] = state + ranges_list = ranges_to_state.keys() + ranges_list.sort() + for ranges in ranges_list: + key = self.ranges_to_string(ranges) + state = ranges_to_state[ranges] + file.write(" %s --> State %d\n" % (key, state['number'])) + for key in ('bol', 'eol', 'eof', 'else'): + state = special_to_state.get(key, None) + if state: + file.write(" %s --> State %d\n" % (key, state['number'])) + + def chars_to_ranges(self, char_list): + char_list.sort() + i = 0 + n = len(char_list) + result = [] + while i < n: + c1 = ord(char_list[i]) + c2 = c1 + i = i + 1 + while i < n and ord(char_list[i]) == c2 + 1: + i = i + 1 + c2 = c2 + 1 + result.append((chr(c1), chr(c2))) + return tuple(result) + + def ranges_to_string(self, range_list): + return string.join(map(self.range_to_string, range_list), ",") + + def range_to_string(self, (c1, c2)): + if c1 == c2: + return repr(c1) + else: + return "%s..%s" % (repr(c1), repr(c2)) +## +## (Superseded by Machines.FastMachine) +## +## class StateTableMachine: +## """ +## StateTableMachine is an alternative representation of a Machine +## that can be run more efficiently. +## """ +## initial_states = None # {state_name:state_index} +## states = None # [([state] indexed by char code, Action)] + +## special_map = {'bol':256, 'eol':257, 'eof':258} + +## def __init__(self, m): +## """ +## Initialise StateTableMachine from Machine |m|. +## """ +## initial_states = self.initial_states = {} +## states = self.states = [None] +## old_to_new = {} +## i = 1 +## for old_state in m.states: +## new_state = ([0] * 259, old_state.get_action()) +## states.append(new_state) +## old_to_new[old_state] = i # new_state +## i = i + 1 +## for name, old_state in m.initial_states.items(): +## initial_states[name] = old_to_new[old_state] +## for old_state in m.states: +## new_state_index = old_to_new[old_state] +## new_table = states[new_state_index][0] +## transitions = old_state.transitions +## for c, old_targets in transitions.items(): +## if old_targets: +## old_target = old_targets[0] +## new_target_index = old_to_new[old_target] +## if len(c) == 1: +## a = ord(c) +## else: +## a = self.special_map[c] +## new_table[a] = states[new_target_index] + +## def dump(self, f): +## f.write("Plex.StateTableMachine:\n") +## f.write(" Initial states:\n") +## for name, index in self.initial_states.items(): +## f.write(" %s: State %d\n" % ( +## repr(name), id(self.states[index]))) +## for i in xrange(1, len(self.states)): +## table, action = self.states[i] +## f.write(" State %d:" % i) +## if action: +## f.write("%s" % action) +## f.write("\n") +## f.write(" %s\n" % map(id,table)) + + + + + + diff --git a/addons/wiki/Plex/Regexps.py b/addons/wiki/Plex/Regexps.py new file mode 100755 index 00000000000..02c2eda8341 --- /dev/null +++ b/addons/wiki/Plex/Regexps.py @@ -0,0 +1,557 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Regular Expressions +# +#======================================================================= + +import array +import string +import types +from sys import maxint + +import Errors + +# +# Constants +# + +BOL = 'bol' +EOL = 'eol' +EOF = 'eof' + +nl_code = ord('\n') + +# +# Helper functions +# + +def chars_to_ranges(s): + """ + Return a list of character codes consisting of pairs + [code1a, code1b, code2a, code2b,...] which cover all + the characters in |s|. + """ + char_list = list(s) + char_list.sort() + i = 0 + n = len(char_list) + result = [] + while i < n: + code1 = ord(char_list[i]) + code2 = code1 + 1 + i = i + 1 + while i < n and code2 >= ord(char_list[i]): + code2 = code2 + 1 + i = i + 1 + result.append(code1) + result.append(code2) + return result + +def uppercase_range(code1, code2): + """ + If the range of characters from code1 to code2-1 includes any + lower case letters, return the corresponding upper case range. + """ + code3 = max(code1, ord('a')) + code4 = min(code2, ord('z') + 1) + if code3 < code4: + d = ord('A') - ord('a') + return (code3 + d, code4 + d) + else: + return None + +def lowercase_range(code1, code2): + """ + If the range of characters from code1 to code2-1 includes any + upper case letters, return the corresponding lower case range. + """ + code3 = max(code1, ord('A')) + code4 = min(code2, ord('Z') + 1) + if code3 < code4: + d = ord('a') - ord('A') + return (code3 + d, code4 + d) + else: + return None + +def CodeRanges(code_list): + """ + Given a list of codes as returned by chars_to_ranges, return + an RE which will match a character in any of the ranges. + """ + re_list = [] + for i in xrange(0, len(code_list), 2): + re_list.append(CodeRange(code_list[i], code_list[i + 1])) + return apply(Alt, tuple(re_list)) + +def CodeRange(code1, code2): + """ + CodeRange(code1, code2) is an RE which matches any character + with a code |c| in the range |code1| <= |c| < |code2|. + """ + if code1 <= nl_code < code2: + return Alt(RawCodeRange(code1, nl_code), + RawNewline, + RawCodeRange(nl_code + 1, code2)) + else: + return RawCodeRange(code1, code2) + +# +# Abstract classes +# + +class RE: + """RE is the base class for regular expression constructors. + The following operators are defined on REs: + + re1 + re2 is an RE which matches |re1| followed by |re2| + re1 | re2 is an RE which matches either |re1| or |re2| + """ + + nullable = 1 # True if this RE can match 0 input symbols + match_nl = 1 # True if this RE can match a string ending with '\n' + str = None # Set to a string to override the class's __str__ result + + def build_machine(self, machine, initial_state, final_state, + match_bol, nocase): + """ + This method should add states to |machine| to implement this + RE, starting at |initial_state| and ending at |final_state|. + If |match_bol| is true, the RE must be able to match at the + beginning of a line. If nocase is true, upper and lower case + letters should be treated as equivalent. + """ + raise exceptions.UnimplementedMethod("%s.build_machine not implemented" % + self.__class__.__name__) + + def build_opt(self, m, initial_state, c): + """ + Given a state |s| of machine |m|, return a new state + reachable from |s| on character |c| or epsilon. + """ + s = m.new_state() + initial_state.link_to(s) + initial_state.add_transition(c, s) + return s + + def __add__(self, other): + return Seq(self, other) + + def __or__(self, other): + return Alt(self, other) + + def __str__(self): + if self.str: + return self.str + else: + return self.calc_str() + + def check_re(self, num, value): + if not isinstance(value, RE): + self.wrong_type(num, value, "Plex.RE instance") + + def check_string(self, num, value): + if type(value) <> type(''): + self.wrong_type(num, value, "string") + + def check_char(self, num, value): + self.check_string(num, value) + if len(value) <> 1: + raise Errors.PlexValueError("Invalid value for argument %d of Plex.%s." + "Expected a string of length 1, got: %s" % ( + num, self.__class__.__name__, repr(value))) + + def wrong_type(self, num, value, expected): + if type(value) == types.InstanceType: + got = "%s.%s instance" % ( + value.__class__.__module__, value.__class__.__name__) + else: + got = type(value).__name__ + raise Errors.PlexTypeError("Invalid type for argument %d of Plex.%s " + "(expected %s, got %s" % ( + num, self.__class__.__name__, expected, got)) + +# +# Primitive RE constructors +# ------------------------- +# +# These are the basic REs from which all others are built. +# + +## class Char(RE): +## """ +## Char(c) is an RE which matches the character |c|. +## """ + +## nullable = 0 + +## def __init__(self, char): +## self.char = char +## self.match_nl = char == '\n' + +## def build_machine(self, m, initial_state, final_state, match_bol, nocase): +## c = self.char +## if match_bol and c <> BOL: +## s1 = self.build_opt(m, initial_state, BOL) +## else: +## s1 = initial_state +## if c == '\n' or c == EOF: +## s1 = self.build_opt(m, s1, EOL) +## if len(c) == 1: +## code = ord(self.char) +## s1.add_transition((code, code+1), final_state) +## if nocase and is_letter_code(code): +## code2 = other_case_code(code) +## s1.add_transition((code2, code2+1), final_state) +## else: +## s1.add_transition(c, final_state) + +## def calc_str(self): +## return "Char(%s)" % repr(self.char) + +def Char(c): + """ + Char(c) is an RE which matches the character |c|. + """ + if len(c) == 1: + result = CodeRange(ord(c), ord(c) + 1) + else: + result = SpecialSymbol(c) + result.str = "Char(%s)" % repr(c) + return result + +class RawCodeRange(RE): + """ + RawCodeRange(code1, code2) is a low-level RE which matches any character + with a code |c| in the range |code1| <= |c| < |code2|, where the range + does not include newline. For internal use only. + """ + nullable = 0 + match_nl = 0 + range = None # (code, code) + uppercase_range = None # (code, code) or None + lowercase_range = None # (code, code) or None + + def __init__(self, code1, code2): + self.range = (code1, code2) + self.uppercase_range = uppercase_range(code1, code2) + self.lowercase_range = lowercase_range(code1, code2) + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + if match_bol: + initial_state = self.build_opt(m, initial_state, BOL) + initial_state.add_transition(self.range, final_state) + if nocase: + if self.uppercase_range: + initial_state.add_transition(self.uppercase_range, final_state) + if self.lowercase_range: + initial_state.add_transition(self.lowercase_range, final_state) + + def calc_str(self): + return "CodeRange(%d,%d)" % (self.code1, self.code2) + +class _RawNewline(RE): + """ + RawNewline is a low-level RE which matches a newline character. + For internal use only. + """ + nullable = 0 + match_nl = 1 + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + if match_bol: + initial_state = self.build_opt(m, initial_state, BOL) + s = self.build_opt(m, initial_state, EOL) + s.add_transition((nl_code, nl_code + 1), final_state) + +RawNewline = _RawNewline() + + +class SpecialSymbol(RE): + """ + SpecialSymbol(sym) is an RE which matches the special input + symbol |sym|, which is one of BOL, EOL or EOF. + """ + nullable = 0 + match_nl = 0 + sym = None + + def __init__(self, sym): + self.sym = sym + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + # Sequences 'bol bol' and 'bol eof' are impossible, so only need + # to allow for bol if sym is eol + if match_bol and self.sym == EOL: + initial_state = self.build_opt(m, initial_state, BOL) + initial_state.add_transition(self.sym, final_state) + + +class Seq(RE): + """Seq(re1, re2, re3...) is an RE which matches |re1| followed by + |re2| followed by |re3|...""" + + def __init__(self, *re_list): + nullable = 1 + for i in xrange(len(re_list)): + re = re_list[i] + self.check_re(i, re) + nullable = nullable and re.nullable + self.re_list = re_list + self.nullable = nullable + i = len(re_list) + match_nl = 0 + while i: + i = i - 1 + re = re_list[i] + if re.match_nl: + match_nl = 1 + break + if not re.nullable: + break + self.match_nl = match_nl + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + re_list = self.re_list + if len(re_list) == 0: + initial_state.link_to(final_state) + else: + s1 = initial_state + n = len(re_list) + for i in xrange(n): + if i < n - 1: + s2 = m.new_state() + else: + s2 = final_state + re = re_list[i] + re.build_machine(m, s1, s2, match_bol, nocase) + s1 = s2 + match_bol = re.match_nl or (match_bol and re.nullable) + + def calc_str(self): + return "Seq(%s)" % string.join(map(str, self.re_list), ",") + + +class Alt(RE): + """Alt(re1, re2, re3...) is an RE which matches either |re1| or + |re2| or |re3|...""" + + def __init__(self, *re_list): + self.re_list = re_list + nullable = 0 + match_nl = 0 + nullable_res = [] + non_nullable_res = [] + i = 1 + for re in re_list: + self.check_re(i, re) + if re.nullable: + nullable_res.append(re) + nullable = 1 + else: + non_nullable_res.append(re) + if re.match_nl: + match_nl = 1 + i = i + 1 + self.nullable_res = nullable_res + self.non_nullable_res = non_nullable_res + self.nullable = nullable + self.match_nl = match_nl + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + for re in self.nullable_res: + re.build_machine(m, initial_state, final_state, match_bol, nocase) + if self.non_nullable_res: + if match_bol: + initial_state = self.build_opt(m, initial_state, BOL) + for re in self.non_nullable_res: + re.build_machine(m, initial_state, final_state, 0, nocase) + + def calc_str(self): + return "Alt(%s)" % string.join(map(str, self.re_list), ",") + + +class Rep1(RE): + """Rep1(re) is an RE which matches one or more repetitions of |re|.""" + + def __init__(self, re): + self.check_re(1, re) + self.re = re + self.nullable = re.nullable + self.match_nl = re.match_nl + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + s1 = m.new_state() + s2 = m.new_state() + initial_state.link_to(s1) + self.re.build_machine(m, s1, s2, match_bol or self.re.match_nl, nocase) + s2.link_to(s1) + s2.link_to(final_state) + + def calc_str(self): + return "Rep1(%s)" % self.re + + +class SwitchCase(RE): + """ + SwitchCase(re, nocase) is an RE which matches the same strings as RE, + but treating upper and lower case letters according to |nocase|. If + |nocase| is true, case is ignored, otherwise it is not. + """ + re = None + nocase = None + + def __init__(self, re, nocase): + self.re = re + self.nocase = nocase + self.nullable = re.nullable + self.match_nl = re.match_nl + + def build_machine(self, m, initial_state, final_state, match_bol, nocase): + self.re.build_machine(m, initial_state, final_state, match_bol, + self.nocase) + + def calc_str(self): + if self.nocase: + name = "NoCase" + else: + name = "Case" + return "%s(%s)" % (name, self.re) + +# +# Composite RE constructors +# ------------------------- +# +# These REs are defined in terms of the primitive REs. +# + +Empty = Seq() +Empty.__doc__ = \ + """ + Empty is an RE which matches the empty string. + """ +Empty.str = "Empty" + +def Str1(s): + """ + Str1(s) is an RE which matches the literal string |s|. + """ + result = apply(Seq, tuple(map(Char, s))) + result.str = "Str(%s)" % repr(s) + return result + +def Str(*strs): + """ + Str(s) is an RE which matches the literal string |s|. + Str(s1, s2, s3, ...) is an RE which matches any of |s1| or |s2| or |s3|... + """ + if len(strs) == 1: + return Str1(strs[0]) + else: + result = apply(Alt, tuple(map(Str1, strs))) + result.str = "Str(%s)" % string.join(map(repr, strs), ",") + return result + +def Any(s): + """ + Any(s) is an RE which matches any character in the string |s|. + """ + #result = apply(Alt, tuple(map(Char, s))) + result = CodeRanges(chars_to_ranges(s)) + result.str = "Any(%s)" % repr(s) + return result + +def AnyBut(s): + """ + AnyBut(s) is an RE which matches any character (including + newline) which is not in the string |s|. + """ + ranges = chars_to_ranges(s) + ranges.insert(0, -maxint) + ranges.append(maxint) + result = CodeRanges(ranges) + result.str = "AnyBut(%s)" % repr(s) + return result + +AnyChar = AnyBut("") +AnyChar.__doc__ = \ + """ + AnyChar is an RE which matches any single character (including a newline). + """ +AnyChar.str = "AnyChar" + +def Range(s1, s2 = None): + """ + Range(c1, c2) is an RE which matches any single character in the range + |c1| to |c2| inclusive. + Range(s) where |s| is a string of even length is an RE which matches + any single character in the ranges |s[0]| to |s[1]|, |s[2]| to |s[3]|,... + """ + if s2: + result = CodeRange(ord(s1), ord(s2) + 1) + result.str = "Range(%s,%s)" % (s1, s2) + else: + ranges = [] + for i in range(0, len(s1), 2): + ranges.append(CodeRange(ord(s1[i]), ord(s1[i+1]) + 1)) + result = apply(Alt, tuple(ranges)) + result.str = "Range(%s)" % repr(s1) + return result + +def Opt(re): + """ + Opt(re) is an RE which matches either |re| or the empty string. + """ + result = Alt(re, Empty) + result.str = "Opt(%s)" % re + return result + +def Rep(re): + """ + Rep(re) is an RE which matches zero or more repetitions of |re|. + """ + result = Opt(Rep1(re)) + result.str = "Rep(%s)" % re + return result + +def NoCase(re): + """ + NoCase(re) is an RE which matches the same strings as RE, but treating + upper and lower case letters as equivalent. + """ + return SwitchCase(re, nocase = 1) + +def Case(re): + """ + Case(re) is an RE which matches the same strings as RE, but treating + upper and lower case letters as distinct, i.e. it cancels the effect + of any enclosing NoCase(). + """ + return SwitchCase(re, nocase = 0) + +# +# RE Constants +# + +Bol = Char(BOL) +Bol.__doc__ = \ + """ + Bol is an RE which matches the beginning of a line. + """ +Bol.str = "Bol" + +Eol = Char(EOL) +Eol.__doc__ = \ + """ + Eol is an RE which matches the end of a line. + """ +Eol.str = "Eol" + +Eof = Char(EOF) +Eof.__doc__ = \ + """ + Eof is an RE which matches the end of the file. + """ +Eof.str = "Eof" + diff --git a/addons/wiki/Plex/Scanners.py b/addons/wiki/Plex/Scanners.py new file mode 100755 index 00000000000..6278d88ba4d --- /dev/null +++ b/addons/wiki/Plex/Scanners.py @@ -0,0 +1,377 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# +# Scanning an input stream +# +#======================================================================= + +import Errors +from Regexps import BOL, EOL, EOF + +class Scanner: + """ + A Scanner is used to read tokens from a stream of characters + using the token set specified by a Plex.Lexicon. + + Constructor: + + Scanner(lexicon, stream, name = '') + + See the docstring of the __init__ method for details. + + Methods: + + See the docstrings of the individual methods for more + information. + + read() --> (value, text) + Reads the next lexical token from the stream. + + position() --> (name, line, col) + Returns the position of the last token read using the + read() method. + + begin(state_name) + Causes scanner to change state. + + produce(value [, text]) + Causes return of a token value to the caller of the + Scanner. + + """ + + lexicon = None # Lexicon + stream = None # file-like object + name = '' + buffer = '' + buf_start_pos = 0 # position in input of start of buffer + next_pos = 0 # position in input of next char to read + cur_pos = 0 # position in input of current char + cur_line = 1 # line number of current char + cur_line_start = 0 # position in input of start of current line + start_pos = 0 # position in input of start of token + start_line = 0 # line number of start of token + start_col = 0 # position in line of start of token + text = None # text of last token read + initial_state = None # Node + state_name = '' # Name of initial state + queue = None # list of tokens to be returned + trace = 0 + + def __init__(self, lexicon, stream, name = ''): + """ + Scanner(lexicon, stream, name = '') + + |lexicon| is a Plex.Lexicon instance specifying the lexical tokens + to be recognised. + + |stream| can be a file object or anything which implements a + compatible read() method. + + |name| is optional, and may be the name of the file being + scanned or any other identifying string. + """ + self.lexicon = lexicon + self.stream = stream + self.name = name + self.queue = [] + self.initial_state = None + self.begin('') + self.next_pos = 0 + self.cur_pos = 0 + self.cur_line_start = 0 + self.cur_char = BOL + self.input_state = 1 + + def read(self): + """ + Read the next lexical token from the stream and return a + tuple (value, text), where |value| is the value associated with + the token as specified by the Lexicon, and |text| is the actual + string read from the stream. Returns (None, '') on end of file. + """ + queue = self.queue + while not queue: + self.text, action = self.scan_a_token() + if action is None: + self.produce(None) + self.eof() + else: + value = action.perform(self, self.text) + if value is not None: + self.produce(value) + result = queue[0] + del queue[0] + return result + + def scan_a_token(self): + """ + Read the next input sequence recognised by the machine + and return (text, action). Returns ('', None) on end of + file. + """ + self.start_pos = self.cur_pos + self.start_line = self.cur_line + self.start_col = self.cur_pos - self.cur_line_start +# if self.trace: +# action = self.run_machine() +# else: +# action = self.run_machine_inlined() + action = self.run_machine_inlined() + if action: + if self.trace: + print "Scanner: read: Performing", action, "%d:%d" % ( + self.start_pos, self.cur_pos) + base = self.buf_start_pos + text = self.buffer[self.start_pos - base : self.cur_pos - base] + return (text, action) + else: + if self.cur_pos == self.start_pos: + if self.cur_char == EOL: + self.next_char() + if not self.cur_char or self.cur_char == EOF: + return ('', None) + raise Errors.UnrecognizedInput(self, self.state_name) + + def run_machine(self): + """ + Run the machine until no more transitions are possible. + """ + self.state = self.initial_state + self.backup_state = None + while self.transition(): + pass + return self.back_up() + + def run_machine_inlined(self): + """ + Inlined version of run_machine for speed. + """ + state = self.initial_state + cur_pos = self.cur_pos + cur_line = self.cur_line + cur_line_start = self.cur_line_start + cur_char = self.cur_char + input_state = self.input_state + next_pos = self.next_pos + buffer = self.buffer + buf_start_pos = self.buf_start_pos + buf_len = len(buffer) + backup_state = None + trace = self.trace + while 1: + if trace: #TRACE# + print "State %d, %d/%d:%s -->" % ( #TRACE# + state['number'], input_state, cur_pos, repr(cur_char)), #TRACE# + # Begin inlined self.save_for_backup() + #action = state.action #@slow + action = state['action'] #@fast + if action: + backup_state = ( + action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos) + # End inlined self.save_for_backup() + c = cur_char + #new_state = state.new_state(c) #@slow + new_state = state.get(c, -1) #@fast + if new_state == -1: #@fast + new_state = c and state.get('else') #@fast + if new_state: + if trace: #TRACE# + print "State %d" % new_state['number'] #TRACE# + state = new_state + # Begin inlined: self.next_char() + if input_state == 1: + cur_pos = next_pos + # Begin inlined: c = self.read_char() + buf_index = next_pos - buf_start_pos + if buf_index < buf_len: + c = buffer[buf_index] + next_pos = next_pos + 1 + else: + discard = self.start_pos - buf_start_pos + data = self.stream.read(0x1000) + buffer = self.buffer[discard:] + data + self.buffer = buffer + buf_start_pos = buf_start_pos + discard + self.buf_start_pos = buf_start_pos + buf_len = len(buffer) + buf_index = buf_index - discard + if data: + c = buffer[buf_index] + next_pos = next_pos + 1 + else: + c = '' + # End inlined: c = self.read_char() + if c == '\n': + cur_char = EOL + input_state = 2 + elif not c: + cur_char = EOL + input_state = 4 + else: + cur_char = c + elif input_state == 2: + cur_char = '\n' + input_state = 3 + elif input_state == 3: + cur_line = cur_line + 1 + cur_line_start = cur_pos = next_pos + cur_char = BOL + input_state = 1 + elif input_state == 4: + cur_char = EOF + input_state = 5 + else: # input_state = 5 + cur_char = '' + # End inlined self.next_char() + else: # not new_state + if trace: #TRACE# + print "blocked" #TRACE# + # Begin inlined: action = self.back_up() + if backup_state: + (action, cur_pos, cur_line, cur_line_start, + cur_char, input_state, next_pos) = backup_state + else: + action = None + break # while 1 + # End inlined: action = self.back_up() + self.cur_pos = cur_pos + self.cur_line = cur_line + self.cur_line_start = cur_line_start + self.cur_char = cur_char + self.input_state = input_state + self.next_pos = next_pos + if trace: #TRACE# + if action: #TRACE# + print "Doing", action #TRACE# + return action + +# def transition(self): +# self.save_for_backup() +# c = self.cur_char +# new_state = self.state.new_state(c) +# if new_state: +# if self.trace: +# print "Scanner: read: State %d: %s --> State %d" % ( +# self.state.number, repr(c), new_state.number) +# self.state = new_state +# self.next_char() +# return 1 +# else: +# if self.trace: +# print "Scanner: read: State %d: %s --> blocked" % ( +# self.state.number, repr(c)) +# return 0 + +# def save_for_backup(self): +# action = self.state.get_action() +# if action: +# if self.trace: +# print "Scanner: read: Saving backup point at", self.cur_pos +# self.backup_state = ( +# action, self.cur_pos, self.cur_line, self.cur_line_start, +# self.cur_char, self.input_state, self.next_pos) + +# def back_up(self): +# backup_state = self.backup_state +# if backup_state: +# (action, self.cur_pos, self.cur_line, self.cur_line_start, +# self.cur_char, self.input_state, self.next_pos) = backup_state +# if self.trace: +# print "Scanner: read: Backing up to", self.cur_pos +# return action +# else: +# return None + + def next_char(self): + input_state = self.input_state + if self.trace: + print "Scanner: next:", " "*20, "[%d] %d" % (input_state, self.cur_pos), + if input_state == 1: + self.cur_pos = self.next_pos + c = self.read_char() + if c == '\n': + self.cur_char = EOL + self.input_state = 2 + elif not c: + self.cur_char = EOL + self.input_state = 4 + else: + self.cur_char = c + elif input_state == 2: + self.cur_char = '\n' + self.input_state = 3 + elif input_state == 3: + self.cur_line = self.cur_line + 1 + self.cur_line_start = self.cur_pos = self.next_pos + self.cur_char = BOL + self.input_state = 1 + elif input_state == 4: + self.cur_char = EOF + self.input_state = 5 + else: # input_state = 5 + self.cur_char = '' + if self.trace: + print "--> [%d] %d %s" % (input_state, self.cur_pos, repr(self.cur_char)) + +# def read_char(self): +# """ +# Get the next input character, filling the buffer if necessary. +# Returns '' at end of file. +# """ +# next_pos = self.next_pos +# buf_index = next_pos - self.buf_start_pos +# if buf_index == len(self.buffer): +# discard = self.start_pos - self.buf_start_pos +# data = self.stream.read(0x1000) +# self.buffer = self.buffer[discard:] + data +# self.buf_start_pos = self.buf_start_pos + discard +# buf_index = buf_index - discard +# if not data: +# return '' +# c = self.buffer[buf_index] +# self.next_pos = next_pos + 1 +# return c + + def position(self): + """ + Return a tuple (name, line, col) representing the location of + the last token read using the read() method. |name| is the + name that was provided to the Scanner constructor; |line| + is the line number in the stream (1-based); |col| is the + position within the line of the first character of the token + (0-based). + """ + return (self.name, self.start_line, self.start_col) + + def begin(self, state_name): + """Set the current state of the scanner to the named state.""" + self.initial_state = ( + self.lexicon.get_initial_state(state_name)) + self.state_name = state_name + + def produce(self, value, text = None): + """ + Called from an action procedure, causes |value| to be returned + as the token value from read(). If |text| is supplied, it is + returned in place of the scanned text. + + produce() can be called more than once during a single call to an action + procedure, in which case the tokens are queued up and returned one + at a time by subsequent calls to read(), until the queue is empty, + whereupon scanning resumes. + """ + if text is None: + text = self.text + self.queue.append((value, text)) + + def eof(self): + """ + Override this method if you want something to be done at + end of file. + """ + +# For backward compatibility: +setattr(Scanner, "yield", Scanner.produce) diff --git a/addons/wiki/Plex/Timing.py b/addons/wiki/Plex/Timing.py new file mode 100755 index 00000000000..f47c5c89de7 --- /dev/null +++ b/addons/wiki/Plex/Timing.py @@ -0,0 +1,22 @@ +# +# Get time in platform-dependent way +# + +import os +from sys import platform, exit, stderr + +if platform == 'mac': + import MacOS + def time(): + return MacOS.GetTicks() / 60.0 + timekind = "real" +elif hasattr(os, 'times'): + def time(): + t = os.times() + return t[0] + t[1] + timekind = "cpu" +else: + stderr.write( + "Don't know how to get time on platform %s\n" % repr(platform)) + exit(1) + diff --git a/addons/wiki/Plex/Traditional.py b/addons/wiki/Plex/Traditional.py new file mode 100755 index 00000000000..b3148c1ec0e --- /dev/null +++ b/addons/wiki/Plex/Traditional.py @@ -0,0 +1,154 @@ +#======================================================================= +# +# Python Lexical Analyser +# +# Traditional Regular Expression Syntax +# +#======================================================================= + +from Regexps import * +from Errors import PlexError + +class RegexpSyntaxError(PlexError): + pass + +def re(s): + """ + Convert traditional string representation of regular expression |s| + into Plex representation. + """ + return REParser(s).parse_re() + +class REParser: + + def __init__(self, s): + self.s = s + self.i = -1 + self.end = 0 + self.next() + + def parse_re(self): + re = self.parse_alt() + if not self.end: + self.error("Unexpected %s" % repr(self.c)) + return re + + def parse_alt(self): + """Parse a set of alternative regexps.""" + re = self.parse_seq() + if self.c == '|': + re_list = [re] + while self.c == '|': + self.next() + re_list.append(self.parse_seq()) + re = apply(Alt, tuple(re_list)) + return re + + def parse_seq(self): + """Parse a sequence of regexps.""" + re_list = [] + while not self.end and not self.c in "|)": + re_list.append(self.parse_mod()) + return apply(Seq, tuple(re_list)) + + def parse_mod(self): + """Parse a primitive regexp followed by *, +, ? modifiers.""" + re = self.parse_prim() + while not self.end and self.c in "*+?": + if self.c == '*': + re = Rep(re) + elif self.c == '+': + re = Rep1(re) + else: # self.c == '?' + re = Opt(re) + self.next() + return re + + def parse_prim(self): + """Parse a primitive regexp.""" + c = self.get() + if c == '.': + re = AnyBut("\n") + elif c == '^': + re = Bol + elif c == '$': + re = Eol + elif c == '(': + re = self.parse_alt() + self.expect(')') + elif c == '[': + re = self.parse_charset() + self.expect(']') + else: + if c == '\\': + c = self.get() + re = Char(c) + return re + + def parse_charset(self): + """Parse a charset. Does not include the surrounding [].""" + char_list = [] + invert = 0 + if self.c == '^': + invert = 1 + self.next() + if self.c == ']': + char_list.append(']') + self.next() + while not self.end and self.c <> ']': + c1 = self.get() + if self.c == '-' and self.lookahead(1) <> ']': + self.next() + c2 = self.get() + for a in xrange(ord(c1), ord(c2) + 1): + char_list.append(chr(a)) + else: + char_list.append(c1) + chars = string.join(char_list, "") + if invert: + return AnyBut(chars) + else: + return Any(chars) + + def next(self): + """Advance to the next char.""" + s = self.s + i = self.i = self.i + 1 + if i < len(s): + self.c = s[i] + else: + self.c = '' + self.end = 1 + + def get(self): + if self.end: + self.error("Premature end of string") + c = self.c + self.next() + return c + + def lookahead(self, n): + """Look ahead n chars.""" + j = self.i + n + if j < len(self.s): + return self.s[j] + else: + return '' + + def expect(self, c): + """ + Expect to find character |c| at current position. + Raises an exception otherwise. + """ + if self.c == c: + self.next() + else: + self.error("Missing %s" % repr(c)) + + def error(self, mess): + """Raise exception to signal syntax error in regexp.""" + raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( + repr(self.s), self.i, mess)) + + + diff --git a/addons/wiki/Plex/Transitions.py b/addons/wiki/Plex/Transitions.py new file mode 100755 index 00000000000..5b0d4fca6cf --- /dev/null +++ b/addons/wiki/Plex/Transitions.py @@ -0,0 +1,252 @@ +# +# Plex - Transition Maps +# +# This version represents state sets direcly as dicts +# for speed. +# + +from copy import copy +import string +from sys import maxint +from types import TupleType + +class TransitionMap: + """ + A TransitionMap maps an input event to a set of states. + An input event is one of: a range of character codes, + the empty string (representing an epsilon move), or one + of the special symbols BOL, EOL, EOF. + + For characters, this implementation compactly represents + the map by means of a list: + + [code_0, states_0, code_1, states_1, code_2, states_2, + ..., code_n-1, states_n-1, code_n] + + where |code_i| is a character code, and |states_i| is a + set of states corresponding to characters with codes |c| + in the range |code_i| <= |c| <= |code_i+1|. + + The following invariants hold: + n >= 1 + code_0 == -maxint + code_n == maxint + code_i < code_i+1 for i in 0..n-1 + states_0 == states_n-1 + + Mappings for the special events '', BOL, EOL, EOF are + kept separately in a dictionary. + """ + + map = None # The list of codes and states + special = None # Mapping for special events + + def __init__(self, map = None, special = None): + if not map: + map = [-maxint, {}, maxint] + if not special: + special = {} + self.map = map + self.special = special + #self.check() ### + + def add(self, event, new_state, + TupleType = TupleType): + """ + Add transition to |new_state| on |event|. + """ + if type(event) == TupleType: + code0, code1 = event + i = self.split(code0) + j = self.split(code1) + map = self.map + while i < j: + map[i + 1][new_state] = 1 + i = i + 2 + else: + self.get_special(event)[new_state] = 1 + + def add_set(self, event, new_set, + TupleType = TupleType): + """ + Add transitions to the states in |new_set| on |event|. + """ + if type(event) == TupleType: + code0, code1 = event + i = self.split(code0) + j = self.split(code1) + map = self.map + while i < j: + map[i + 1].update(new_set) + i = i + 2 + else: + self.get_special(event).update(new_set) + + def get_epsilon(self): + """ + Return the mapping for epsilon, or None. + """ + return self.special.get('') + + def items(self, + len = len): + """ + Return the mapping as a list of ((code1, code2), state_set) and + (special_event, state_set) pairs. + """ + result = [] + map = self.map + else_set = map[1] + i = 0 + n = len(map) - 1 + code0 = map[0] + while i < n: + set = map[i + 1] + code1 = map[i + 2] + if set or else_set: + result.append(((code0, code1), set)) + code0 = code1 + i = i + 2 + for event, set in self.special.items(): + if set: + result.append((event, set)) + return result + + # ------------------- Private methods -------------------- + + def split(self, code, + len = len, maxint = maxint): + """ + Search the list for the position of the split point for |code|, + inserting a new split point if necessary. Returns index |i| such + that |code| == |map[i]|. + """ + # We use a funky variation on binary search. + map = self.map + hi = len(map) - 1 + # Special case: code == map[-1] + if code == maxint: + return hi + # General case + lo = 0 + # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 + while hi - lo >= 4: + # Find midpoint truncated to even index + mid = ((lo + hi) / 2) & ~1 + if code < map[mid]: + hi = mid + else: + lo = mid + # map[lo] <= code < map[hi] and hi - lo == 2 + if map[lo] == code: + return lo + else: + map[hi:hi] = [code, map[hi - 1].copy()] + #self.check() ### + return hi + + def get_special(self, event): + """ + Get state set for special event, adding a new entry if necessary. + """ + special = self.special + set = special.get(event, None) + if not set: + set = {} + special[event] = set + return set + + # --------------------- Conversion methods ----------------------- + + def __str__(self): + map_strs = [] + map = self.map + n = len(map) + i = 0 + while i < n: + code = map[i] + if code == -maxint: + code_str = "-inf" + elif code == maxint: + code_str = "inf" + else: + code_str = str(code) + map_strs.append(code_str) + i = i + 1 + if i < n: + map_strs.append(state_set_str(map[i])) + i = i + 1 + special_strs = {} + for event, set in self.special.items(): + special_strs[event] = state_set_str(set) + return "[%s]+%s" % ( + string.join(map_strs, ","), + special_strs + ) + + # --------------------- Debugging methods ----------------------- + + def check(self): + """Check data structure integrity.""" + if not self.map[-3] < self.map[-1]: + print self + assert 0 + + def dump(self, file): + map = self.map + i = 0 + n = len(map) - 1 + while i < n: + self.dump_range(map[i], map[i + 2], map[i + 1], file) + i = i + 2 + for event, set in self.special.items(): + if set: + if not event: + event = 'empty' + self.dump_trans(event, set, file) + + def dump_range(self, code0, code1, set, file): + if set: + if code0 == -maxint: + if code1 == maxint: + k = "any" + else: + k = "< %s" % self.dump_char(code1) + elif code1 == maxint: + k = "> %s" % self.dump_char(code0 - 1) + elif code0 == code1 - 1: + k = self.dump_char(code0) + else: + k = "%s..%s" % (self.dump_char(code0), + self.dump_char(code1 - 1)) + self.dump_trans(k, set, file) + + def dump_char(self, code): + if 0 <= code <= 255: + return repr(chr(code)) + else: + return "chr(%d)" % code + + def dump_trans(self, key, set, file): + file.write(" %s --> %s\n" % (key, self.dump_set(set))) + + def dump_set(self, set): + return state_set_str(set) + +# +# State set manipulation functions +# + +#def merge_state_sets(set1, set2): +# for state in set2.keys(): +# set1[state] = 1 + +def state_set_str(set): + state_list = set.keys() + str_list = [] + for state in state_list: + str_list.append("S%d" % state.number) + return "[%s]" % string.join(str_list, ",") + + + diff --git a/addons/wiki/Plex/__init__.py b/addons/wiki/Plex/__init__.py new file mode 100755 index 00000000000..22b9bba3b8c --- /dev/null +++ b/addons/wiki/Plex/__init__.py @@ -0,0 +1,40 @@ +#======================================================================= +# +# Python Lexical Analyser +# +#======================================================================= + +""" +The Plex module provides lexical analysers with similar capabilities +to GNU Flex. The following classes and functions are exported; +see the attached docstrings for more information. + + Scanner For scanning a character stream under the + direction of a Lexicon. + + Lexicon For constructing a lexical definition + to be used by a Scanner. + + Str, Any, AnyBut, AnyChar, Seq, Alt, Opt, Rep, Rep1, + Bol, Eol, Eof, Empty + + Regular expression constructors, for building pattern + definitions for a Lexicon. + + State For defining scanner states when creating a + Lexicon. + + TEXT, IGNORE, Begin + + Actions for associating with patterns when + creating a Lexicon. +""" + +from Actions import TEXT, IGNORE, Begin +from Lexicons import Lexicon, State +from Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range +from Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase +from Scanners import Scanner + + + diff --git a/addons/wiki/Plex/test_tm.py b/addons/wiki/Plex/test_tm.py new file mode 100755 index 00000000000..e08c9e68f93 --- /dev/null +++ b/addons/wiki/Plex/test_tm.py @@ -0,0 +1,24 @@ +import sys +sys.stderr = sys.stdout + +from TransitionMaps import TransitionMap + +m = TransitionMap() +print m + +def add(c, s): + print + print "adding", repr(c), "-->", repr(s) + m.add_transition(c, s) + print m + print "keys:", m.keys() + +add('a','alpha') +add('e', 'eta') +add('f', 'foo') +add('i', 'iota') +add('i', 'imp') +add('eol', 'elephant') + + + diff --git a/addons/wiki/Wiki2Html.py b/addons/wiki/Wiki2Html.py new file mode 100755 index 00000000000..0d84d59af25 --- /dev/null +++ b/addons/wiki/Wiki2Html.py @@ -0,0 +1,332 @@ +# Copyright (C) 2006 Samuel Abels, http://debain.org +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import HTMLParser, re, sys +from WikiParser import WikiParser +from cgi import escape + +class Wiki2Html: + def __init__(self): + self.html = '' + self.buffer = '' + self.counter = 0 + self.in_list = False + self.in_numbered_list = False + self.in_heading = False + self.in_table = False + self.in_cell = False + self.indent_level = 0 + self.url_handler = None + self.wiki_word_handler = None + + + def set_url_handler(self, handler): + """ + Whenever the parser finds a URL in the output, by default it + automatically translates it into a clickable link in the HTML output. + However, if you want more fine-grained control over which URLs are + converted into links, you might want to use this function. + It calls the given handler function before a link is added into the + HTML. The function is passed the URL and the caption. + The handler can the modify the URL and the caption, and pass it + back to the parser. Example handler: + + def url_handler(url, caption): + return (url, 'My Caption') + + The parser uses the returned values to build a link in the HTML. + If URL is None, the item is not linked at all. + """ + self.url_handler = handler + + + def set_wiki_word_handler(self, handler): + """ + Like set_url_handler() but for WikiWords. + Example handler: + + def wiki_word_handler(word): + return (url, word) + + The parser uses the returned values to build a link in the HTML. + If URL is None, the item is not linked at all. + """ + self.wiki_word_handler = handler + + #onclick="openWindow(getURL('/attachment', {model: 'base.wiki', id: id=$id}), {name : 'Attachments'}) + def image(self, text): + image = text.strip() + url = image.split(':')[1] + self.buffer += """ + Please upload Image
\n""" + + def indent(self, level): + self.indent_level = level + + def dedent(self, level): + self.indent_level = level + + + def code_start(self, text): + self.buffer += '
'
+
+
+    def code(self, text):
+        self.buffer += escape(text)
+
+
+    def code_end(self, text):
+        self.buffer += '
\n' + + + def title1_start(self, text): + self.buffer += '

' + + + def title1_end(self, text): + self.buffer += '

' + + + def title2_start(self, text): + self.buffer += '

' + + + def title2_end(self, text): + self.buffer += '

' + + + def title3_start(self, text): + self.buffer += '

' + + + def title3_end(self, text): + self.buffer += '

' + + + def italic_start(self, text): + self.buffer += '' + + + def italic_end(self, text): + self.buffer += '' + + + def bold_start(self, text): + self.buffer += '' + + + def bold_end(self, text): + self.buffer += '' + + + def underline_start(self, text): + self.buffer += '' + + + def underline_end(self, text): + self.buffer += '' + + + def unnumbered_list_start(self, text): + self.buffer += '\n' + self.in_list = False + + + def list_item_start(self, text): + self.buffer += '
  • ' + + + def list_item_end(self, text): + self.__flush(True) + self.buffer += '
    • \n' + + + def table_start(self, text): + self.in_table = True + self.counter = 0 + self.buffer += '\n' + + + def heading_start(self, text): + self.in_heading = True + self.row_start(text) + + + def row_start(self, text): + self.buffer += '' + + + def cell_start(self, text): + self.in_cell = True + attribs = '' + if len(text) - 1 > 1: + attribs += ' colspan=%i' % (len(text) - 1) + if self.in_heading: + self.buffer += '' % attribs + else: + self.buffer += '' % attribs + self.__flush(True) + + + def cell_end(self, text): + # If the cell contains only a hash, replace it by a counter. + if self.buffer.strip() == '#': + self.counter += 1 + self.buffer = str(self.counter) + '.' + self.__flush(True) + self.in_cell = False + if self.in_heading: + self.html += '' + else: + self.html += '' + self.html += '\n' + (' ' * self.indent_level) + + + def row_end(self, text): + self.buffer += '\n' + (' ' * self.indent_level) + + + def heading_end(self, text): + self.in_heading = False + self.row_end(text) + + + def table_end(self, text): + self.in_table = False + self.buffer += '
    \n' + + + def text(self, text): + self.buffer += escape(text).replace('\n', '
    \n') + + + def newline(self, text): + if (self.in_table + or self.in_list + or (self.buffer[-5:-2] == '') + or (self.buffer[-6:-3] == '')): + self.buffer += '\n' + return + self.buffer += '
    \n' + + + def link(self, text): + text = text[1:-1] + pos = text.find('-') + if pos == -1: + url = text + caption = url + else: + url = text[:pos] + caption = text[pos + 1:] + if self.url_handler is not None: + (url, caption) = self.url_handler(url.strip()+"'", caption) + url = escape(url) + caption = escape(caption) + if url is None: + self.buffer += caption + else: + if url.strip().startswith('http') or url.strip().startswith('www') or url.strip().startswith('https') or url.strip().startswith('frp'): + self.buffer += '%s' % (url.strip(), caption) + else: + self.buffer += '%s' % ("/wiki?id="+url.strip()+"&title='" + caption.strip() + "'" , caption ) + + def wiki_word(self, text): + if text.startswith('->'): + text = text[2:] + url = text + caption = text + if self.wiki_word_handler is not None: + (url, caption) = self.wiki_word_handler("/wiki?id='"+url.strip()+"'", caption) + url = escape(url) + caption = escape(caption) + if url is None: + self.buffer += caption + else: + self.buffer += '%s' % ("/wiki?id='"+url.strip()+"'", caption) + + + def not_wiki_word(self, text): + self.buffer += escape(text) + + + def __flush(self, strip = False): + if strip: + self.html += self.buffer.strip() + else: + self.html += self.buffer + self.buffer = '' + + + def read(self, filename): + infile = open(filename, 'U') + parser = WikiParser(infile, filename) + nonecount = 0 + self.html = '' + while True: + token = parser.read() + position = parser.position() + if token[0] is None: + nonecount += 1 # This is because Plex is broken. + if nonecount >= 2: + break + try: + method = getattr(self, token[0]) + except: + method = None + if method is not None: + method(token[1]) + infile.close() + self.__flush() + + +if __name__ == '__main__': + import unittest + + class Wiki2HtmlTest(unittest.TestCase): + def runTest(self): + # Read the entire file into one string. + filename = 'markup.txt' + infile = open(filename, 'r') + in_str = infile.read() + infile.close() + + # Parse the file. + parser = Wiki2Html() + parser.read(filename) + html = parser.html + print html + + # Now *that's* a poor test, huh? For a better test, look + # at the test in Html2Wiki.py. + assert len(html) > 1000 + + testcase = Wiki2HtmlTest() + runner = unittest.TextTestRunner() + runner.run(testcase) diff --git a/addons/wiki/WikiParser.py b/addons/wiki/WikiParser.py new file mode 100755 index 00000000000..ea4fbbe9ccb --- /dev/null +++ b/addons/wiki/WikiParser.py @@ -0,0 +1,417 @@ +# Copyright (C) 2006 Samuel Abels, http://debain.org +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import sys, string +sys.path.append('..') +from Plex import * + +# Single char definitions. +letter = Range('AZaz') +digit = Range('09') +spaces = Any(" \t\r\n") +nl = Str("\n") | Eof +not_nl = AnyBut("\r\n") +dash = Str('-') +colon = Str(':') +hash = Str('#') +at = Str('@') +equal = Str('=') +star = Str('*') +slash = Str('/') +o_bracket = Str('{') +c_bracket = Str('}') +underscore = Str('_') +dot = Str('.') +punctuation = Any('.!?,;:') +quotes = Any('\'"') + +# Single word definitions. +name = letter + Rep(letter | digit) +file_name = Rep1(letter | digit | dash | underscore | dot) +wiki_force = Str('->') + name +wiki_word = Range('AZ') + name + Range('AZ') + name +not_w_word = Str('!') + wiki_word +indentation = Rep(Str(' ')) | Rep(Str("\t")) +variable = letter + Rep(letter | digit | Str('_')) + Rep(letter | digit) +proto = Alt(Str('http'), Str('https'), Str('ftp'), Str('mailto')) +login = file_name + Opt(colon + file_name) +host = name + Rep(letter | digit | dash | underscore | dot) + name +path = Opt(slash) + file_name + Rep(slash + file_name) + Opt(slash) +arg_value = Rep1(letter | digit | dash | underscore | dot | Str('%')) +arg = file_name + equal + Opt(arg_value) +args = Str('?') + arg + Rep(Str('&') + arg) +path_args = path + Opt(args) +url = proto + colon + Str('//') + Opt(login + at) + host + Opt(path_args) +code = Str('#Text') + nl +code_end = Str('#End') + nl + +# Markup. +line = Rep(not_nl) + nl +blank_line = indentation + nl +words = Rep1(letter | digit | spaces) +list_item = Bol + Alt(hash, star) + Str(' ') +italic_start = Alt(Bol, spaces) + slash +italic_end = slash + Alt(Eol, spaces, punctuation) +phrase = Rep1(words | dash | at | punctuation | slash | quotes| hash | Str('$')) +title1 = equal + phrase + equal +title2 = equal + equal + phrase + equal + equal +title3 = equal + equal + equal + phrase + equal + equal + equal +heading = Str('#Heading') + nl +row = Str('#Row') + nl +cell = Rep1(Str('|')) + Str(' ') +internal_link = Str('[') + url + Opt(spaces + phrase) + Str(']') +external_link = Str('[') + path_args + Opt(spaces + phrase) + Str(']') +image = Str('Attach') + colon + Opt(phrase) + nl + +class WikiParser(Scanner): + def __init__(self, file, filename): + Scanner.__init__(self, self.lexicon, file, filename) + self.my_buffer = '' + self.indentation_stack = [0] + self.bracket_nesting_level = 0 + self.in_italic = False + self.in_bold = False + self.in_underline = False + self.in_list = False + self.in_list_item = False + self.in_table = False + self.in_heading = False + self.in_row = False + self.in_cell = False + self.begin('indent') + + def _buffer_flush(self): + indent = self._get_current_indent_level() + #print "_buffer_flush (%i): '%s'" % (indent, self.my_buffer) + if self.my_buffer == '': + return + if self.my_buffer == '\n': + self.produce('newline', '\n') + else: + self.produce('text', self.my_buffer) + #print "Flush produced" + self.my_buffer = '' + + def _newline_action(self, text): + #print '_newline_action' + self._buffer_flush() + self.produce('newline', text) + if self.bracket_nesting_level == 0: + self.begin('indent') + + def _indentation_action(self, text): + #print '_indentation_action' + self.my_buffer += text + self._buffer_flush() + current_level = self._get_current_indent_level() + new_level = len(text) + if new_level > current_level: + self._indent_to(new_level) + elif new_level < current_level: + self._dedent_to(new_level) + self.begin('') + + def _indent_to(self, new_level): + self.indentation_stack.append(new_level) + self.produce('indent', new_level) + + def _dedent_to(self, new_level): + while new_level < self._get_current_indent_level(): + self.indentation_stack.pop() + self.produce('dedent', self._get_current_indent_level()) + + def _get_current_indent_level(self): + return self.indentation_stack[-1] + + def _code_start(self, text): + #print '_code_start: ' + text + self._buffer_flush() + self.produce('code_start', text) + self.begin('code') + + def _code(self, text): + #print '_code: ' + text + self.my_buffer += text + + def _code_end(self, text): + #print '_code_end: ' + text + if self.my_buffer != '': + self.produce('code', self.my_buffer) + self.my_buffer = '' + self.produce('code_end', text) + self.begin('') + + def _title1(self, text): + #print '_title1: ' + text + self._buffer_flush() + self.produce('title1_start', text[0]) + self.my_buffer += text[1:-1] + self._buffer_flush() + self.produce('title1_end', text[-1]) + + def _title2(self, text): + #print '_title2: ' + text + self._buffer_flush() + self.produce('title2_start', text[0:2]) + self.my_buffer += text[2:-2] + self._buffer_flush() + self.produce('title2_end', text[-2:]) + + def _title3(self, text): + #print '_title3: ' + text + self._buffer_flush() + self.produce('title3_start', text[0:3]) + self.my_buffer += text[3:-3] + self._buffer_flush() + self.produce('title3_end', text[-3:]) + + def _italic_start(self, text): + #print '_italic_start "%s"' % text + if text[0] != '/': + self.my_buffer += text[0] + self._buffer_flush() + self.in_italic = True + self.produce('italic_start', text[-1]) + + def _italic_end(self, text): + #print '_italic_end' + self._buffer_flush() + self.produce('italic_end', text[0]) + self.in_italic = False + self.my_buffer += text[1] + + def _bold(self, text): + #print '_bold' + self._buffer_flush() + if self.in_bold: + self.produce('bold_end', text) + self.in_bold = False + else: + self.in_bold = True + self.produce('bold_start', text) + + def _underline(self, text): + #print '_underline' + self._buffer_flush() + if self.in_underline: + self.produce('underline_end', text) + self.in_underline = False + else: + self.in_underline = True + self.produce('underline_start', text) + + def _list_item(self, text): + self._buffer_flush() + if not self.in_list: + if text.startswith('#'): + self.produce('numbered_list_start', '') + elif text.startswith('*'): + self.produce('unnumbered_list_start', '') + self.in_list = True + if self.in_list_item: + self._list_item_end() + self.in_list_item = True + self.produce('list_item_start') + + def _list_item_end(self): + self.in_list_item = False + self.produce('list_item_end', '') + + def _list_end(self): + if self.in_list_item: + self._list_item_end() + self.in_list = False + self.produce('list_end', '') + + def _heading_start(self, text): + self._buffer_flush() + if not self.in_table: + self.produce('table_start', '') + self.in_table = True + if self.in_heading: + self._heading_end() + self.in_heading = True + self.produce('heading_start', text[:-1]) + self._newline_action(text[-1]) + + def _row_start(self, text): + self._buffer_flush() + if not self.in_table: + self.produce('table_start', '') + self.in_table = True + if self.in_heading: + self._heading_end() + if self.in_row: + self._row_end() + self.in_row = True + self.produce('row_start', text[:-1]) + self._newline_action(text[-1]) + + def _cell_start(self, text): + self._buffer_flush() + if not self.in_heading and not self.in_row: + self._text(text) + return + if self.in_cell: + self._cell_end() + self.in_cell = True + self.produce('cell_start') + + def _cell_end(self): + self.in_cell = False + self.produce('cell_end', '') + + def _row_end(self): + if self.in_cell: + self._cell_end() + self.in_row = False + self.produce('row_end', '') + + def _heading_end(self): + if self.in_cell: + self._cell_end() + self.in_heading = False + self.produce('heading_end', '') + + def _table_end(self): + if self.in_heading: + self._heading_end() + if self.in_row: + self._row_end() + self.in_table = False + self.produce('table_end', '') + + def _link(self, text): + self._buffer_flush() + self.produce('link', text) + + def _image(self, text): + self._buffer_flush() + self.produce('image', text) + + def _wiki_word(self, text): + self._buffer_flush() + self.produce('wiki_word', text) + + def _not_wiki_word(self, text): + self._buffer_flush() + self.produce('not_wiki_word_intro', text[0]) + self.produce('not_wiki_word', text[1:]) + + def _open_bracket_action(self, text): + #print '_open_bracket_action' + self.my_buffer += text + self.bracket_nesting_level = self.bracket_nesting_level + 1 + + def _close_bracket_action(self, text): + #print '_close_bracket_action' + self.my_buffer += text + self.bracket_nesting_level = self.bracket_nesting_level - 1 + + def _blank_line(self, text): + self._dedent_to(0) + self.my_buffer += text + if self.in_list: + self._list_end() + if self.in_table: + self._table_end() + self._buffer_flush() + + def _text(self, text): + #print "Char: '%s'" % text + self.my_buffer += text + + def eof(self): + #print "EOF" + self._blank_line('') + + lexicon = Lexicon([ + # Handle whitespace and indentation. + (nl, _newline_action), + State('indent', [ + (blank_line, _blank_line), + (indentation, _indentation_action) + ]), + + # Preformatted text/code. + (code, _code_start), + State('code', [ + (code_end, _code_end), + (AnyChar, _code) + ]), + + # Brackets. + (o_bracket, _open_bracket_action), + (c_bracket, _close_bracket_action), + + # Styles. + (list_item, _list_item), + (title1, _title1), + (title2, _title2), + (title3, _title3), + (star, _bold), + (underscore, _underline), + (italic_start, _italic_start), + (italic_end, _italic_end), + + # Tables. + (heading, _heading_start), + (row, _row_start), + (cell, _cell_start), + + # Other. + (internal_link, _link), + (external_link, _link), + (not_w_word, _not_wiki_word), + (wiki_word, _wiki_word), + (wiki_force, _wiki_word), + (AnyChar, _text), + (image, _image) + ]) + + +if __name__ == '__main__': + import unittest + + class ParserTest(unittest.TestCase): + def runTest(self): + # Read the entire file into one string. + filename = 'markup.txt' + infile = open(filename, "U") + in_text = infile.read() + infile.close() + + # Re-open and parse the entire file. + infile = open(filename, "r") + scanner = WikiParser(infile, filename) + content = '' + nonecount = 0 + while True: + token = scanner.read() + position = scanner.position() + if token[0] is None: + nonecount += 1 # This is because Plex is broken. + if nonecount >= 2: + break + #print "Token type: %s, Token: '%s'" % (token[0], token[1]) + if not token[0] in ['indent', 'dedent']: + content += token[1] + + # Make sure that every single string was extracted. + #print content + assert content == in_text + + testcase = ParserTest() + runner = unittest.TextTestRunner() + runner.run(testcase) diff --git a/addons/wiki/__init__.py b/addons/wiki/__init__.py new file mode 100755 index 00000000000..5001d3c5ac7 --- /dev/null +++ b/addons/wiki/__init__.py @@ -0,0 +1,10 @@ +__all__ = ['Wiki2Html', + 'Html2Wiki'] + +from Wiki2Html import Wiki2Html +from Html2Wiki import Html2Wiki +import wiki +#import IndexServer +import os + +os.system("python IndexServer.py") \ No newline at end of file diff --git a/addons/wiki/__terp__.py b/addons/wiki/__terp__.py new file mode 100755 index 00000000000..831610cf640 --- /dev/null +++ b/addons/wiki/__terp__.py @@ -0,0 +1,12 @@ +{ + "name" : "Wiki Base System", + "version" : "1.0", + "author" : "Axelor Sarl", + "category" : "Generic Modules/Others", + "depends" : ["base", "project"], + "init_xml" : [], + "demo_xml" : [], + "update_xml" : ["wiki_view.xml","wiki_data.xml"], + "active": False, + "installable": True +} \ No newline at end of file diff --git a/addons/wiki/config.py b/addons/wiki/config.py new file mode 100755 index 00000000000..ce33e652e5e --- /dev/null +++ b/addons/wiki/config.py @@ -0,0 +1,7 @@ +import os + +index_server = "" +index_port = 8000 + +tiny_port = 8069 +tiny_host = 'localhost' diff --git a/addons/wiki/wiki.py b/addons/wiki/wiki.py new file mode 100755 index 00000000000..a04f671aeee --- /dev/null +++ b/addons/wiki/wiki.py @@ -0,0 +1,423 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2004-2006 TINY SPRL. (http://axelor.com) All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +from osv import fields, osv +import time +from Wiki2Html import Wiki2Html +import xmlrpclib +import base64 +import os +from StringIO import StringIO +import popen2 + +class Tag(osv.osv): + _name="base.wiki.tag" + _description="Wiki" + _rec_name="title" + _columns={ + 'title':fields.char('Title',size=128), + } +Tag() + +class Wiki(osv.osv): + _name="base.wiki" + _description="Wiki" + _rec_name="title" + _order = 'model_id' + _columns={ + 'title':fields.char('Title', size=128, select=True), + 'last_modify_by':fields.many2one('res.users',"Last Modify By"), + 'text_area':fields.text("Content", select=True), + 'wiki_create_uid':fields.many2one('res.users','Authour', select=True), + 'wiki_create_date':fields.datetime("Created on", select=True), + 'wiki_write_date':fields.datetime("Last modified", select=True), + 'tags':fields.many2many("base.wiki.tag","wiki_tag_many_id","wiki_id","tag_id","Tags", select=True), + #'forum_id':fields.one2many('base.wiki.forum','wiki_id','Forum Lines'), + 'history_id':fields.one2many('base.wiki.history','history_wiki_id','History Lines'), + 'html':fields.text("Html Data", select=True), + 'path':fields.char('Page Path',size=128), + 'model_id': fields.many2one('ir.model', 'Model id', select=True, ondelete='cascade'), + 'res_id': fields.integer("Record Id"), + 'minor_edit':fields.boolean('Thisd is a minor edit', select=True), + 'summary':fields.char('Summary',size=256, select=True), + } + _defaults = { + 'wiki_create_date': lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'), + 'wiki_create_uid': lambda obj,cr,uid,context: uid, + } + + def __init__(self, cr, pool): + super(Wiki, self).__init__(cr, pool) + self.oldmodel = None + + def read(self, cr, uid, cids, fields=None, context=None, load='_classic_read'): + result = super(Wiki, self).read(cr, uid, cids, {}, load='_classic_read') + + if context and context.get('index'): + ids = self.search(cr, uid, [('model_id','!=',False), ('id','in',cids)]) + res = self.read(cr, uid, ids) + + buffer = '' + for rs in res: + if self.oldmodel != rs['model_id'][1] or self.oldmodel == None: + self.oldmodel = rs['model_id'][1] + buffer+= '\n==' + self.pool.get('ir.model').browse(cr, uid, rs['model_id'][0]).name + '==\n' + + rec_url = '/form/view?model=' + rs['model_id'][1] + "&id=" + str(rs['res_id']) + edit_url = '/form/edit?model=' + rs['model_id'][1] + "&id=" + str(rs['res_id']) + buffer+= '#Row\n' + buffer+= '| # | ' + rs['title'] + ' | [' + rs['path'] + ' - wiki]' + ' | [' + rec_url + ' - Browse ] | [' + edit_url + ' - Edit ]\n' + + ids = self.search(cr, uid, [('model_id','=',False), ('id','in',cids)]) + res = self.read(cr, uid, ids) + + if res: + buffer+= '\n==Other Pages==\n' + for rs in res: + + buffer+= '* [' + rs['path'] + ' - ' + rs['title'] + ']\n' + + return [{'html':self.Wiki2Html(buffer)}] + else: + return result + + def Wiki2Html(self, wiki): + fileName = 'data.txt' + file = open(fileName, 'w') + file.write(wiki+'\n\n') + file.close() + + parser = Wiki2Html() + parser.read(fileName) + return parser.html + + def create(self, cr, uid, vals, context=None): + if vals.get('text_area'): + vals['html'] = self.Wiki2Html(vals['text_area']) + else: + vals['text_area'] = 'Your text goes here' + vals['html'] = 'You have not create the page.' + if not vals.has_key('minor_edit'): + return super(Wiki,self).create(cr, uid, vals, context) + vals['history_id']=[[0,0,{'minor_edit':vals['minor_edit'],'text_area':vals['text_area'],'modify_by':uid,'summary':vals['summary']}]] + return super(Wiki,self).create(cr, uid, vals, context) + + def write(self, cr, uid, ids, vals, context=None): + vals['wiki_write_date']=time.strftime('%Y-%m-%d %H:%M:%S') + vals['last_modify_by']=uid + wiki_data=self.read(cr,uid,ids,['minor_edit','summary'])[0] + if vals.get('text_area'): + vals['html'] = self.Wiki2Html(vals['text_area']) + if vals.has_key('minor_edit') and vals.has_key('summary'): + vals['history_id']=[[0,0,{'minor_edit':vals['minor_edit'],'text_area':vals['text_area'],'modify_by':uid,'summary':vals['summary']}]] + elif vals.has_key('minor_edit'): + vals['history_id']=[[0,0,{'minor_edit':vals['minor_edit'],'text_area':vals['text_area'],'modify_by':uid,'summary':wiki_data['summary']}]] + elif vals.has_key('summary'): + vals['history_id']=[[0,0,{'minor_edit':wiki_data['summary'],'text_area':vals['text_area'],'modify_by':uid,'summary':vals['summary']}]] + else: + vals['history_id']=[[0,0,{'minor_edit':wiki_data['minor_edit'],'text_area':vals['text_area'],'modify_by':uid,'summary':wiki_data['summary']}]] + return super(Wiki,self).write(cr, uid, ids, vals, context) +Wiki() + +#class Forum(osv.osv): +# _name="base.wiki.forum" +# _description="Wiki Forum" +# _rec_name="title" +# _columns={ +# 'title':fields.char('Title',size=128), +# 'replies':fields.integer('Replies'), +# 'wiki_authour':fields.many2one('res.users','Created By'), +# 'last_post':fields.datetime("Last Post",readonly=True), +# 'by':fields.many2one('res.users','By',readonly=True), +# 'discussion_lines':fields.one2many('base.wiki.discussion','forum_id','Discussion Lines'), +# 'wiki_id':fields.many2one('base.wiki','Wiki Id') +# } +#Forum() +# +#class Discussion(osv.osv): +# _name="base.wiki.discussion" +# _description="Wiki Discussion" +# _rec_name="wiki_authour" +# _columns={ +# 'wiki_authour':fields.many2one('res.users','Authour',readonly=True), +# 'posted_date':fields.datetime("Posted Date",readonly=True), +# 'message':fields.text('Message'), +# 'forum_id':fields.many2one('base.wiki.forum','Forum Id') +# } +#Discussion() + +class History(osv.osv): + _name="base.wiki.history" + _description="Wiki History" + _rec_name="date_time" + _order = 'id DESC' + _columns={ + 'date_time':fields.datetime("Date",select=True), + 'text_area':fields.text("Text area",select=True), + 'minor_edit':fields.boolean('This is a major edit ?',select=True), + 'summary':fields.char('Summary',size=256, select=True), + 'modify_by':fields.many2one('res.users',"Modify By", select=True), + 'hist_write_date':fields.datetime("Last modified", select=True), + 'history_wiki_id':fields.many2one('base.wiki','Wiki Id', select=True) + } + _defaults = { + 'hist_write_date': lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'), + 'modify_by': lambda obj,cr,uid,context: uid, + } + + def getDiff(self, cr, uid, v1, v2, context={}): + import difflib + + history_pool = self.pool.get('base.wiki.history') + + text1 = history_pool.read(cr, uid, [v1], ['text_area'])[0]['text_area'] + text2 = history_pool.read(cr, uid, [v2], ['text_area'])[0]['text_area'] + + line1 = text1.splitlines(1) + line2 = text2.splitlines(1) + + diff = difflib.HtmlDiff() + + return diff.make_file(line1, line2, "Revision-%s" % (v1), "Revision-%s" % (v2), context=False) + +History() + +from StringIO import StringIO +from HTMLParser import HTMLParser + +class IndexLine(osv.osv): + _name="base.index.line" + _description="Index Lines" + _columns={ + 'name':fields.text('Content', select=True), + 'active':fields.boolean('Active', select=True), + 'model':fields.char('Resource',size=256, select=True), + 'res_id':fields.integer('Resource Id', select=True), + } + _defaults = { + 'active': lambda *a: True, + } + + def __init__(self, cr , pool): + super(IndexLine, self).__init__(cr, pool) + + def reIndex(self, cr, uid, ids, context={}): + password = self.pool.get('res.users').browse(cr, uid, uid).password + self.server.init(cr.dbname, uid, password, True) + self.server.open() + + self.Index(cr, uid, context) + + self.server.close() + return True + + def Index(self, cr, uid, ids, context={}): + + entry_pool = self.pool.get('base.index.line') + eids = entry_pool.search(cr, uid, []) +# entry_pool.unlink(cr, uid, eids) +# cr.commit() + + model_pool = self.pool.get('ir.model') + field_pool = self.pool.get('ir.model.fields') + + ids = model_pool.search(cr, uid, [('model','!=','base.index.line')]) + models = model_pool.read(cr, uid, ids, ['id', 'model']) + + for mod in models: + res_ids = None + + if str(mod['model']).startswith('workflow'): + continue + + if str(mod['model']).startswith('ir'): + continue + + if str(mod['model']).startswith('base.index.line'): + continue + + if str(mod['model']).startswith('res.currency'): + continue + + if str(mod['model']).startswith('base.wiki'): + continue + + if str(mod['model']).startswith('report'): + continue + + if str(mod['model']).startswith('account_analytic_analysis.summary.user'): + continue + + if str(mod['model']).startswith('hr_timesheet_invoice.factor'): + continue + + try: + fids = field_pool.search(cr, 1, [('model_id','=',mod['id'])]) + fdata = field_pool.read(cr, uid, fids, ['name','relation', 'ttype']) + except Exception, e: + continue +# +# fkeys = {} +# tkeys = {} +# for fd in fdata: +# fkeys[fd['name']] = fd['relation'] +# tkeys[fd['name']] = fd['ttype'] + + res_pool = None + res_ids = None + + res_pool = self.pool.get(mod['model']) + + res_datas = None + + try: + res_ids = res_pool.search(cr, 1, []) + res_datas = res_pool.read(cr, 1, res_ids) + #cr.execute("select * from %s" % ( mod['model'].replace('.','_'))) + #res_datas = cr.dictfetchall() + except Exception, e: + print e + continue + + print 'Indexing : ', mod['model'], ' Auto Status : ' + + if res_datas: + for res_data in res_datas: + + exist_ids = entry_pool.search(cr, uid, [('res_id','=',res_data['id']),('model','=',mod['model'])]) + if exist_ids.__len__() > 0: + continue + + final_res = '' +# +# for col in res_data: +# if fkeys.has_key(col) and fkeys[col] != 'NULL': +# if tkeys.has_key(col) and tkeys[col] == 'one2many': +# rel_pool = self.pool.get(fkeys.get(col)) +# try: +# foreign_key = res_pool._columns[col]._fields_id +# except Exception, e: +# print e +# continue +# fkids = rel_pool.search(cr, 1, [(foreign_key, '=', res_data['id'])]) +# if fkids.__len__() > 0: +# fkdata = rel_pool.read(cr, 1, fkids) +# for fkd in fkdata: +# final_res += ' '.join(map(str,fkd.values())) + + final_res += ' '.join(map(str,res_data.values())) + + entry_pool.create(cr, uid, { + 'model':mod['model'], + 'res_id':res_data['id'], + 'name': final_res + }) + cr.commit() + + return True + + def createIndex(self, cr, uid, ids, context={}): + return self.Index(cr, uid, ids, context) + + def getResult(self, cr, uid, key, reSearch=True, context = {}): + +# password = self.pool.get('res.users').browse(cr, uid, uid).password +# self.server.init(cr.dbname, uid, password, True) + + if context and context.get('read_all') and key: + try: + #res = self.server.search(key) + ids = self.search(cr, uid, [('name','ilike',key)]) + res = self.read(cr, uid, ids, ['model', 'res_id']) + except Exception, e: + raise osv.except_osv('Search Error !!', e.faultString) + + trs = {} + for rs in res: + model = rs.get('model') + id = rs.get('res_id') + if trs.has_key(model): + olds = trs[model] + olds.append(id) + else: + trs[model] = [id] + + res = trs + if res: + buffer = '' + for mod in res.keys(): + sbuff = '' + obj_pool = self.pool.get(mod) + header = False + + for id in res[mod]: + name = None + try: + name = obj_pool.name_get(cr, 1, [id])[0][1] + obj_pool.read(cr, uid, [id]) + header = True + except Exception, e: + header = False + continue + + rec_url = '/form/view?model=' + mod + "&id=" + str(id) + edit_url = '/form/edit?model=' + mod + "&id=" + str(id) + sbuff+= '#Row\n' + name = name.replace('/',' ') + name = name.replace('*',' ') + name = name.replace('_',' ') + sbuff+= '| # | ' + name + ' | [' + mod + '/' + str(id) + ' - Wiki]' + ' | [' + rec_url + ' - Browse ] | [' + edit_url + ' - Edit ]\n' + + if header == True: + mid = self.pool.get('ir.model').search(cr, uid, [('model','=',mod)])[0] + mod_data = self.pool.get('ir.model').browse(cr, uid, mid).name + sbuff = '\n==' + mod_data + '==\n' + sbuff + + buffer += sbuff + html = self.pool.get('base.wiki').Wiki2Html(buffer) + html = html.replace('$s', '') + html = html.replace('$es', '') + return [{'html':html}] +# else: +# #TODO: need to create an index on the new records +# if reSearch: +# self.Index(cr, uid, context) +# self.getResult(cr, uid, key, False, context) + + return None + +IndexLine() + +class Project(osv.osv): + _inherit = 'project.project' + _name = 'project.project' + _columns = { + 'wiki':fields.many2one('base.wiki', 'Wiki'), + 'dwiki':fields.many2one('base.wiki', 'Developer Wiki'), + } +Project() diff --git a/addons/wiki/wiki_data.xml b/addons/wiki/wiki_data.xml new file mode 100755 index 00000000000..447dcf374f2 --- /dev/null +++ b/addons/wiki/wiki_data.xml @@ -0,0 +1,12 @@ + + + + + Wiki Home + home + False + + + + + \ No newline at end of file diff --git a/addons/wiki/wiki_view.xml b/addons/wiki/wiki_view.xml new file mode 100755 index 00000000000..43299fedcaa --- /dev/null +++ b/addons/wiki/wiki_view.xml @@ -0,0 +1,249 @@ + + + + + base.wiki.tree + base.wiki + tree + + + + + + + + + + + + base.wiki.form + base.wiki + form + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +     +     + + + Wiki Pages + base.wiki + form + form,tree + + + + + + + + base.wiki.forum.tree + base.wiki.forum + tree + + + + + + + + + + + + + base.wiki.history.tree + base.wiki.history + tree + + + + + + + + + + + + base.wiki.history.form + base.wiki.history + form + +
    + + + + + + + +
    + + + Wiki History + base.wiki.history + form + form,tree + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + base.index.line.form + base.index.line + form + +
    + + + +