From f3e33d3ae65068776623e98b4b2a2ad623e690d2 Mon Sep 17 00:00:00 2001 From: Lucas Hoffmann Date: Mon, 9 Jan 2023 20:18:54 +0100 Subject: [PATCH 1/5] dictdDBs.wordnet: convert python script to python 3 --- pkgs/servers/dict/wordnet_structures.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/servers/dict/wordnet_structures.py b/pkgs/servers/dict/wordnet_structures.py index 6a88427c5da1..72d91e64ab42 100644 --- a/pkgs/servers/dict/wordnet_structures.py +++ b/pkgs/servers/dict/wordnet_structures.py @@ -262,7 +262,7 @@ original version.\n\n self.dict_entry_write(file_index, file_data, '00-database-url', '00-database-url\n%s\n' % self.wn_url) - words = self.word_data.keys() + words = list(self.word_data.keys()) words.sort() for word in words: for wi in self.word_data[word]: @@ -306,14 +306,14 @@ if (__name__ == '__main__'): wnd = WordnetDict(wn_url=options.wn_url, desc_short=options.desc_short, desc_long=options.desc_long) for i in range(0,len(args),2): - print 'Opening index file %r...' % args[i] + print('Opening index file %r...' % args[i]) file_index = file(args[i]) - print 'Opening data file %r...' % args[i+1] + print('Opening data file %r...' % args[i+1]) file_data = file(args[i+1]) - print 'Parsing index file and data file...' + print('Parsing index file and data file...') wnd.wn_dict_add(file_index, file_data) - print 'All input files parsed. Writing output to index file %r and data file %r.' % (options.oi, options.od) + print('All input files parsed. Writing output to index file %r and data file %r.' % (options.oi, options.od)) wnd.dict_generate(file(options.oi, 'w'),file(options.od, 'w')) - print 'All done.' + print('All done.') From b5c88c02d18cf75a6a55d4c6c86804dd36866c09 Mon Sep 17 00:00:00 2001 From: Lucas Hoffmann Date: Mon, 9 Jan 2023 20:26:02 +0100 Subject: [PATCH 2/5] dictdDBs.wordnet: use python3 --- pkgs/servers/dict/dictd-wordnet.nix | 4 +- pkgs/servers/dict/wordnet_structures.py | 58 ++++++++++++------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/pkgs/servers/dict/dictd-wordnet.nix b/pkgs/servers/dict/dictd-wordnet.nix index 8378102dac53..663bd54897bf 100644 --- a/pkgs/servers/dict/dictd-wordnet.nix +++ b/pkgs/servers/dict/dictd-wordnet.nix @@ -1,10 +1,10 @@ -{lib, stdenv, python2, wordnet, writeScript}: +{lib, stdenv, python3, wordnet, writeScript}: stdenv.mkDerivation rec { version = "542"; pname = "dict-db-wordnet"; - buildInputs = [python2 wordnet]; + buildInputs = [python3 wordnet]; convert = ./wordnet_structures.py; builder = writeScript "builder.sh" '' diff --git a/pkgs/servers/dict/wordnet_structures.py b/pkgs/servers/dict/wordnet_structures.py index 72d91e64ab42..7aada092baad 100644 --- a/pkgs/servers/dict/wordnet_structures.py +++ b/pkgs/servers/dict/wordnet_structures.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #Copyright 2007 Sebastian Hagen # This file is part of wordnet_tools. @@ -49,7 +49,7 @@ class WordIndex: self.ptrs = ptrs self.synsets = synsets self.tagsense_count = tagsense_count - + @classmethod def build_from_line(cls, line_data, synset_map): line_split = line_data.split() @@ -61,14 +61,14 @@ class WordIndex: tagsense_count = int(line_split[5 + ptr_count],10) synsets = [synset_map[int(line_split[i],10)] for i in range(6 + ptr_count, 6 + ptr_count + synset_count)] return cls(lemma, category, ptrs, synsets, tagsense_count) - + @classmethod def build_from_file(cls, f, synset_map, rv_base=None): if (rv_base is None): rv = {} else: rv = rv_base - + for line in f: if (line.startswith(' ')): continue @@ -81,8 +81,8 @@ class WordIndex: def __repr__(self): return '%s%s' % (self.__class__.__name__, (self.lemma, self.category, self.ptrs, self.synsets, self.tagsense_count)) - - + + class WordIndexDictFormatter(WordIndex): category_map_rev = { CAT_NOUN: 'n', @@ -96,12 +96,12 @@ class WordIndexDictFormatter(WordIndex): prefix_fmtn_line_first = ' ' prefix_fmtf_line_nonfirst = '%5d: ' prefix_fmtn_line_nonfirst = ' ' - + def dict_str(self): tw = TextWrapper(width=self.LINE_WIDTH_MAX, initial_indent=(self.prefix_fmtf_line_first % self.category_map_rev[self.category]), subsequent_indent=self.prefix_fmtn_line_first) - + lines = (tw.wrap(self.synsets[0].dict_str())) i = 2 for synset in self.synsets[1:]: @@ -122,7 +122,7 @@ class Synset: self.gloss = gloss self.frames = frames self.comments = [] - + @classmethod def build_from_line(cls, line_data): line_split = line_data.split() @@ -132,7 +132,7 @@ class Synset: words = [line_split[i] for i in range(4, 4 + word_count*2,2)] ptr_count = int(line_split[4 + word_count*2],10) ptrs = [(line_split[i], line_split[i+1], line_split[i+2], line_split[i+3]) for i in range(5 + word_count*2,4 + word_count*2 + ptr_count*4,4)] - + tok = line_split[5 + word_count*2 + ptr_count*4] base = 6 + word_count*2 + ptr_count*4 if (tok != '|'): @@ -141,20 +141,20 @@ class Synset: base += frame_count*3 + 1 else: frames = [] - + line_split2 = line_data.split(None, base) if (len(line_split2) < base): gloss = None else: gloss = line_split2[-1] - + return cls(synset_offset, ss_type, words, ptrs, gloss, frames) - + @classmethod def build_from_file(cls, f): rv = {} comments = [] - + for line in f: if (line.startswith(' ')): line_s = line.lstrip().rstrip('\n') @@ -197,14 +197,14 @@ original version.\n\n datetime_fmt = '%Y-%m-%dT%H:%M:%S' base64_map = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' - + def __init__(self, wn_url, desc_short, desc_long): self.word_data = {} self.wn_url = wn_url self.desc_short = desc_short self.desc_long = desc_long self.wn_license = None - + def wn_dict_add(self, file_index, file_data): file_data.seek(0) file_index.seek(0) @@ -212,7 +212,7 @@ original version.\n\n WordIndexDictFormatter.build_from_file(file_index, synsets, self.word_data) if (license_lines): self.wn_license = '\n'.join(license_lines) + '\n' - + @classmethod def base64_encode(cls, i): """Encode a non-negative integer into a dictd compatible base64 string""" @@ -223,7 +223,7 @@ original version.\n\n while (r < i): e += 1 r = 64**e - 1 - + rv = '' while (e > 0): e -= 1 @@ -231,7 +231,7 @@ original version.\n\n rv += cls.base64_map[d] i = i % (64**e) return rv - + @classmethod def dict_entry_write(cls, file_index, file_data, key, entry, linesep='\n'): """Write a single dict entry for to index and data files""" @@ -240,7 +240,7 @@ original version.\n\n entry_len = len(entry) file_index.write('%s\t%s\t%s%s' % (key, cls.base64_encode(entry_start), cls.base64_encode(entry_len), linesep)) - + def dict_generate(self, file_index, file_data): file_index.seek(0) file_data.seek(0) @@ -261,7 +261,7 @@ original version.\n\n self.dict_entry_write(file_index, file_data, '00-database-short', '00-database-short\n%s\n' % self.desc_short) self.dict_entry_write(file_index, file_data, '00-database-url', '00-database-url\n%s\n' % self.wn_url) - + words = list(self.word_data.keys()) words.sort() for word in words: @@ -280,14 +280,14 @@ original version.\n\n else: continue break - + outstr = '' for wi in self.word_data[word]: outstr += wi.dict_str() + '\n' - + outstr = '%s%s%s' % (word_cs, wi.linesep, outstr) self.dict_entry_write(file_index, file_data, word_cs, outstr, wi.linesep) - + file_index.truncate() file_data.truncate() @@ -300,11 +300,11 @@ if (__name__ == '__main__'): op.add_option('--wn_url', dest='wn_url', default='ftp://ftp.cogsci.princeton.edu/pub/wordnet/2.0', help='URL for wordnet sources') op.add_option('--db_desc_short', dest='desc_short', default=' WordNet (r) 2.1 (2005)', help='short dict DB description') op.add_option('--db_desc_long', dest='desc_long', default=' WordNet (r): A Lexical Database for English from the\n Cognitive Science Laboratory at Princeton University', help='long dict DB description') - + (options, args) = op.parse_args() - + wnd = WordnetDict(wn_url=options.wn_url, desc_short=options.desc_short, desc_long=options.desc_long) - + for i in range(0,len(args),2): print('Opening index file %r...' % args[i]) file_index = file(args[i]) @@ -314,6 +314,6 @@ if (__name__ == '__main__'): wnd.wn_dict_add(file_index, file_data) print('All input files parsed. Writing output to index file %r and data file %r.' % (options.oi, options.od)) - + wnd.dict_generate(file(options.oi, 'w'),file(options.od, 'w')) - print('All done.') + print('All done.') From 56d90f3fef4a5849ec848d142a280cada3324b18 Mon Sep 17 00:00:00 2001 From: Lucas Hoffmann Date: Mon, 9 Jan 2023 20:33:30 +0100 Subject: [PATCH 3/5] dictdDBs.wordnet: use open() in python3 instead of file() from python2 --- pkgs/servers/dict/wordnet_structures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 pkgs/servers/dict/wordnet_structures.py diff --git a/pkgs/servers/dict/wordnet_structures.py b/pkgs/servers/dict/wordnet_structures.py old mode 100644 new mode 100755 index 7aada092baad..bfb07c53dc6e --- a/pkgs/servers/dict/wordnet_structures.py +++ b/pkgs/servers/dict/wordnet_structures.py @@ -307,13 +307,13 @@ if (__name__ == '__main__'): for i in range(0,len(args),2): print('Opening index file %r...' % args[i]) - file_index = file(args[i]) + file_index = open(args[i]) print('Opening data file %r...' % args[i+1]) - file_data = file(args[i+1]) + file_data = open(args[i+1]) print('Parsing index file and data file...') wnd.wn_dict_add(file_index, file_data) print('All input files parsed. Writing output to index file %r and data file %r.' % (options.oi, options.od)) - wnd.dict_generate(file(options.oi, 'w'),file(options.od, 'w')) + wnd.dict_generate(open(options.oi, 'w'),open(options.od, 'w')) print('All done.') From 01d76ca6c1f2f71d6078551b84fb00d40df9844b Mon Sep 17 00:00:00 2001 From: Lucas Hoffmann Date: Mon, 9 Jan 2023 20:40:28 +0100 Subject: [PATCH 4/5] dictdDBs.wordnet: fix integer devision in python3 --- pkgs/servers/dict/wordnet_structures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/dict/wordnet_structures.py b/pkgs/servers/dict/wordnet_structures.py index bfb07c53dc6e..e5c80b968fc2 100755 --- a/pkgs/servers/dict/wordnet_structures.py +++ b/pkgs/servers/dict/wordnet_structures.py @@ -26,6 +26,7 @@ # written. import datetime +import math from textwrap import TextWrapper CAT_ADJECTIVE = 0 @@ -227,7 +228,7 @@ original version.\n\n rv = '' while (e > 0): e -= 1 - d = (i / 64**e) + d = math.floor(i / 64**e) rv += cls.base64_map[d] i = i % (64**e) return rv From 9287d37501a63a114bc0eb8b1fe33128656c8934 Mon Sep 17 00:00:00 2001 From: Barry Fishman Date: Mon, 9 Jan 2023 15:35:23 -0500 Subject: [PATCH 5/5] dictdDbs.wiktionary: use python3 In pkgs/servers/dict: * wiktionary/default.nix: change python2 -> python3 * wiktionary/latest_version.py: decodes('utf-8') no longer needed * wiktionary/wiktionary2dict.py: 'import codesc' no longer needed --- pkgs/servers/dict/wiktionary/default.nix | 7 +-- .../servers/dict/wiktionary/latest_version.py | 6 +- .../dict/wiktionary/wiktionary2dict.py | 59 +++++++++---------- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/pkgs/servers/dict/wiktionary/default.nix b/pkgs/servers/dict/wiktionary/default.nix index 03f60f694c53..b15ece0e8512 100644 --- a/pkgs/servers/dict/wiktionary/default.nix +++ b/pkgs/servers/dict/wiktionary/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, python2, dict, glibcLocales }: +{ lib, stdenv, fetchurl, python3, dict, glibcLocales }: stdenv.mkDerivation rec { pname = "dict-db-wiktionary"; @@ -9,8 +9,7 @@ stdenv.mkDerivation rec { sha256 = "qsha26LL2513SDtriE/0zdPX1zlnpzk1KKk+R9dSdew="; }; - # script in nixpkgs does not support python2 - nativeBuildInputs = [ python2 dict glibcLocales ]; + nativeBuildInputs = [ python3 dict glibcLocales ]; dontUnpack = true; @@ -18,7 +17,7 @@ stdenv.mkDerivation rec { mkdir -p $out/share/dictd/ cd $out/share/dictd - ${python2.interpreter} -O ${./wiktionary2dict.py} "${src}" + ${python3.interpreter} -O ${./wiktionary2dict.py} "${src}" dictzip wiktionary-en.dict echo en_US.UTF-8 > locale ''; diff --git a/pkgs/servers/dict/wiktionary/latest_version.py b/pkgs/servers/dict/wiktionary/latest_version.py index 2833a1e05b03..1aa767851cee 100644 --- a/pkgs/servers/dict/wiktionary/latest_version.py +++ b/pkgs/servers/dict/wiktionary/latest_version.py @@ -25,18 +25,18 @@ def nix_prefetch_url(url, algo='sha256'): """Prefetches the content of the given URL.""" print(f'nix-prefetch-url {url}') out = subprocess.check_output(['nix-prefetch-url', '--type', algo, url]) - return out.decode('utf-8').rstrip() + return out.rstrip() current_version = subprocess.check_output([ 'nix', 'eval', '--raw', '-f', dirname(abspath(__file__)) + '/../../../..', 'dictdDBs.wiktionary.version', -]).decode('utf-8') +]) parser = WiktionaryLatestVersionParser(current_version) with urlopen('https://dumps.wikimedia.org/enwiktionary/') as resp: - parser.feed(resp.read().decode('utf-8')) + parser.feed(resp.read()) print(parser.latest_version) diff --git a/pkgs/servers/dict/wiktionary/wiktionary2dict.py b/pkgs/servers/dict/wiktionary/wiktionary2dict.py index 8a0210e7254f..a737079bf5c8 100644 --- a/pkgs/servers/dict/wiktionary/wiktionary2dict.py +++ b/pkgs/servers/dict/wiktionary/wiktionary2dict.py @@ -2,7 +2,6 @@ # Based on code from wiktiondict by Greg Hewgill import re import sys -import codecs import os import textwrap import time @@ -48,7 +47,7 @@ class Delimiter: return self.c def Tokenise(s): - s = unicode(s) + s = str(s) stack = [] last = 0 i = 0 @@ -109,17 +108,17 @@ def Tokenise(s): yield s[last:i] def processSub(templates, tokens, args): - t = tokens.next() - if not isinstance(t, unicode): + t = next(tokens) + if not isinstance(t, str): raise SyntaxError name = t - t = tokens.next() + t = next(tokens) default = None if isinstance(t, Delimiter) and t.c == '|': default = "" while True: - t = tokens.next() - if isinstance(t, unicode): + t = next(tokens) + if isinstance(t, str): default += t elif isinstance(t, OpenDouble): default += processTemplateCall(templates, tokens, args) @@ -128,7 +127,7 @@ def processSub(templates, tokens, args): elif isinstance(t, CloseTriple): break else: - print "Unexpected:", t + print("Unexpected:", t) raise SyntaxError() if name in args: return args[name] @@ -142,14 +141,14 @@ def processTemplateCall(templates, tokens, args): template = tokens.next().strip().lower() args = {} a = 1 - t = tokens.next() + t = next(tokens) while True: if isinstance(t, Delimiter): - name = unicode(a) + name = str(a) arg = "" while True: - t = tokens.next() - if isinstance(t, unicode): + t = next(tokens) + if isinstance(t, str): arg += t elif isinstance(t, OpenDouble): arg += processTemplateCall(templates, tokens, args) @@ -163,9 +162,9 @@ def processTemplateCall(templates, tokens, args): name = arg.strip() arg = "" while True: - t = tokens.next() - if isinstance(t, (unicode, Equals)): - arg += unicode(t) + t = next(tokens) + if isinstance(t, (str, Equals)): + arg += str(t) elif isinstance(t, OpenDouble): arg += processTemplateCall(templates, tokens, args) elif isinstance(t, OpenTriple): @@ -181,7 +180,7 @@ def processTemplateCall(templates, tokens, args): elif isinstance(t, CloseDouble): break else: - print "Unexpected:", t + print("Unexpected:", t) raise SyntaxError #print template, args if template[0] == '#': @@ -208,7 +207,7 @@ def processTemplateCall(templates, tokens, args): else: return "" else: - print "Unknown ParserFunction:", template + print("Unknown ParserFunction:", template) sys.exit(1) if template not in templates: return "{{%s}}" % template @@ -225,13 +224,13 @@ def process(templates, s, args = {}): tokens = Tokenise(s) try: while True: - t = tokens.next() + t = next(tokens) if isinstance(t, OpenDouble): r += processTemplateCall(templates, tokens, args) elif isinstance(t, OpenTriple): r += processSub(templates, tokens, args) else: - r += unicode(t) + r += str(t) except StopIteration: pass return r @@ -250,11 +249,11 @@ def test(): 't6': "t2demo|a", } def t(text, expected): - print "text:", text + print("text:", text) s = process(templates, text) if s != expected: - print "got:", s - print "expected:", expected + print("got:", s) + print("expected:", expected) sys.exit(1) t("{{Name-example}}", "I am a template example, my first name is '''{{{firstName}}}''' and my last name is '''{{{lastName}}}'''. You can reference my page at [[{{{lastName}}}, {{{firstName}}}]].") t("{{Name-example | firstName=John | lastName=Smith }}", "I am a template example, my first name is '''John''' and my last name is '''Smith'''. You can reference my page at [[Smith, John]].") @@ -463,7 +462,7 @@ Parts = { 'Verbal noun': "v.n.", } PartsUsed = {} -for p in Parts.keys(): +for p in list(Parts.keys()): PartsUsed[p] = 0 def encode(s): @@ -641,7 +640,7 @@ def formatNormal(word, doc): # r += " "*(depth-1) + word + " (" + p + ")\n\n" r += " "*(depth-1) + section.heading + "\n\n" else: - print >>errors, "Unknown part: (%s) %s" % (word, section.heading) + print("Unknown part: (%s) %s" % (word, section.heading), file=errors) return "" elif depth > posdepth: return "" @@ -709,8 +708,8 @@ class WikiHandler(xml.sax.ContentHandler): if self.element == "text": if self.page: if self.page in self.long: - print self.page, len(self.text) - print + print(self.page, len(self.text)) + print() self.doPage(self.page, self.text) self.page = None self.text = "" @@ -760,8 +759,7 @@ info = """ This file was converted from the original database on: Wiktionary is available under the GNU Free Documentation License. """ % (time.ctime(), os.path.basename(fn)) -errors = codecs.open("mkdict.err", "w", "utf_8") -e = codecs.getencoder("utf_8") +errors = open("mkdict.err", "w") Templates = {} f = os.popen("bunzip2 -c %s" % fn, "r") @@ -769,10 +767,9 @@ xml.sax.parse(f, TemplateHandler()) f.close() f = os.popen("bunzip2 -c %s" % fn, "r") -out = codecs.getwriter("utf_8")( - os.popen("dictfmt -p wiktionary-en --locale en_US.UTF-8 --columns 0 -u http://en.wiktionary.org", "w")) +out = os.popen("dictfmt -p wiktionary-en --locale en_US.UTF-8 --columns 0 -u http://en.wiktionary.org", "w") -out.write(("%%h English Wiktionary\n%s" % info).encode('utf-8')) +out.write("%%h English Wiktionary\n%s" % info) xml.sax.parse(f, WordHandler()) f.close() out.close()