Basic Working Version
Rudimentary version controlled with in-code statements. Fetched contents of kanji.csv from www.tonypottier.info and converted them into a csv. Other csv's are based unicode ranges from wikipedia.master
parent
a32d18c6d9
commit
c94a2ea418
@ -0,0 +1,186 @@
|
|||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
class Alphabet:
|
||||||
|
|
||||||
|
def __init__(self, name: str, file_path: str = "", data: list=None):
|
||||||
|
if not os.path.isfile(file_path) and file_path != "":
|
||||||
|
raise FileNotFoundError("File has not been found")
|
||||||
|
|
||||||
|
self._name = name
|
||||||
|
self._file_path = file_path
|
||||||
|
self._parsed_html = None
|
||||||
|
self._legend = None
|
||||||
|
|
||||||
|
if data is not None:
|
||||||
|
self._data = data
|
||||||
|
return
|
||||||
|
|
||||||
|
if file_path == "":
|
||||||
|
self._data = []
|
||||||
|
print("Warning: Empty Alphabet!")
|
||||||
|
return
|
||||||
|
|
||||||
|
self._stream = open(file_path, "r")
|
||||||
|
file_name, file_extension = os.path.splitext(self._file_path)
|
||||||
|
if file_extension == '.html':
|
||||||
|
self._data = self.import_from_html()
|
||||||
|
elif file_extension == '.csv':
|
||||||
|
self._data = self.import_from_csv()
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise Exception("Unknown file extension!")
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self._data)
|
||||||
|
|
||||||
|
def get_data(self) -> list:
|
||||||
|
return self._data
|
||||||
|
|
||||||
|
def get_name(self) -> str:
|
||||||
|
return self._name
|
||||||
|
|
||||||
|
def get_unique(self):
|
||||||
|
"""
|
||||||
|
Check if each entry is unique
|
||||||
|
|
||||||
|
Uses Unicode number for comparison
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
characters = set()
|
||||||
|
for entry in self._data:
|
||||||
|
characters.add(entry['Decimal'])
|
||||||
|
|
||||||
|
Alphabet.generate_from_numbers(self._name, list(characters))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def generate_from_numbers(name: str, numbers: list):
|
||||||
|
"""
|
||||||
|
Generate alphabet from a list of unicode numbers
|
||||||
|
:param name:
|
||||||
|
:param numbers:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
data = []
|
||||||
|
|
||||||
|
for n in numbers:
|
||||||
|
entry = {}
|
||||||
|
entry['Char'] = chr(n)
|
||||||
|
entry['Unicode (hex)'] = "U+" + hex(n)[2:]
|
||||||
|
entry['Decimal'] = n
|
||||||
|
entry['Learning Order'] = None
|
||||||
|
data.append(entry)
|
||||||
|
|
||||||
|
return Alphabet(name=name, data=data)
|
||||||
|
|
||||||
|
def import_from_csv(self) -> list:
|
||||||
|
"""
|
||||||
|
Import file from csv
|
||||||
|
"""
|
||||||
|
reader = csv.reader(self._stream)
|
||||||
|
self._legend = next(reader)
|
||||||
|
|
||||||
|
data = []
|
||||||
|
for row in reader:
|
||||||
|
entry = {}
|
||||||
|
for i in range(len(row)):
|
||||||
|
entry[self._legend[i]] = row[i]
|
||||||
|
data.append(entry)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def import_from_html(self) -> list:
|
||||||
|
"""
|
||||||
|
Import file as html table of characters
|
||||||
|
"""
|
||||||
|
html = ""
|
||||||
|
for line in self._stream:
|
||||||
|
html += line
|
||||||
|
|
||||||
|
self._parsed_html = BeautifulSoup(html, features="html.parser")
|
||||||
|
entries = self._parsed_html.find_all('tr')
|
||||||
|
|
||||||
|
self._legend = []
|
||||||
|
for legend_data in entries[0].find_all('td'):
|
||||||
|
self._legend.append(legend_data.string)
|
||||||
|
|
||||||
|
data = []
|
||||||
|
for entry in entries[1:]:
|
||||||
|
parsed_table_data = entry.find_all('td')
|
||||||
|
|
||||||
|
table_data = {}
|
||||||
|
for i in range(len(parsed_table_data)):
|
||||||
|
table_data[self._legend[i]] = parsed_table_data[i].string
|
||||||
|
data.append(table_data)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def export_csv(self, file_path: str):
|
||||||
|
assert(file_path != self._file_path)
|
||||||
|
|
||||||
|
file_name, file_extension = os.path.splitext(file_path)
|
||||||
|
if file_extension != '.csv':
|
||||||
|
file_path = file_name + '.csv'
|
||||||
|
|
||||||
|
stream = open(file_path, 'w')
|
||||||
|
writer = csv.writer(stream)
|
||||||
|
|
||||||
|
writer.writerow(self.get_data()[0].keys())
|
||||||
|
|
||||||
|
for entry in self.get_data():
|
||||||
|
writer.writerow([entry[key] for key in entry.keys()])
|
||||||
|
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
def append_entry(self, unicode_num: int):
|
||||||
|
uni_hex = hex(unicode_num)
|
||||||
|
char = chr(unicode_num)
|
||||||
|
|
||||||
|
print("Appending Chr:{0} Num:{1} Hex:{2} to {3}".format(char, unicode_num, uni_hex, self._name))
|
||||||
|
|
||||||
|
entry = {}
|
||||||
|
entry['Char'] = char
|
||||||
|
entry['Unicode (hex)'] = "U+"+uni_hex[2:]
|
||||||
|
entry['Decimal'] = unicode_num
|
||||||
|
entry['Learning Order'] = None
|
||||||
|
|
||||||
|
self._data.append(entry)
|
||||||
|
|
||||||
|
def append_entry_hex(self, uni_hex: str):
|
||||||
|
self.append_entry(int(uni_hex, 0))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
kanji = Alphabet("Kanji", "kanji.csv")
|
||||||
|
hiragana = Alphabet("Hiragana", "hiragana.csv")
|
||||||
|
katakana = Alphabet("Katakana", "katakana.csv")
|
||||||
|
|
||||||
|
print("Kanji:{0} Hiragana:{1} Katakana:{2}".format(len(kanji.get_data()),
|
||||||
|
len(hiragana.get_data()),
|
||||||
|
len(katakana.get_data(),
|
||||||
|
)))
|
||||||
|
|
||||||
|
# Generate kanji2 from ranges
|
||||||
|
start = 0x4e00
|
||||||
|
end = 0x9fa0
|
||||||
|
|
||||||
|
n = [x for x in range(start, end)]
|
||||||
|
|
||||||
|
start = 0x3400
|
||||||
|
end = 0x4dbf
|
||||||
|
|
||||||
|
n += [x for x in range(start, end)]
|
||||||
|
|
||||||
|
n = list(set(n))
|
||||||
|
|
||||||
|
k2 = Alphabet.generate_from_numbers("Kanji2", n)
|
||||||
|
print("Kanji2: {0}".format(len(k2)))
|
||||||
|
#k2.export_csv("kanji2.csv")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,93 @@
|
|||||||
|
from Alphabet import Alphabet
|
||||||
|
|
||||||
|
|
||||||
|
class Detector:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._alphs = None
|
||||||
|
|
||||||
|
def add_alphabet(self, alph: Alphabet):
|
||||||
|
if self._alphs is None:
|
||||||
|
self._alphs = []
|
||||||
|
self._alphs.append(alph)
|
||||||
|
|
||||||
|
def match_text(self, text: str) -> dict:
|
||||||
|
"""
|
||||||
|
Determine if a given text uses some characters from any alphabet
|
||||||
|
:param text:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
results = {}
|
||||||
|
for alph in self._alphs:
|
||||||
|
results[alph.get_name()] = [[]] # [matches], percentage
|
||||||
|
results['Unknown'] = [[]]
|
||||||
|
|
||||||
|
for c in text:
|
||||||
|
if c == "\n": # ignoring whitespace
|
||||||
|
continue
|
||||||
|
found = False
|
||||||
|
for alph in self._alphs:
|
||||||
|
|
||||||
|
for entry in alph.get_data():
|
||||||
|
if c == entry['Char']:
|
||||||
|
results[alph.get_name()][0].append(c)
|
||||||
|
found = True
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
results['Unknown'][0].append(c)
|
||||||
|
|
||||||
|
# Create statistics
|
||||||
|
for key in results.keys():
|
||||||
|
results[key].append(len(results[key][0])/float(len(text)))
|
||||||
|
return results
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def pretty_result(result: dict, small: bool=False):
|
||||||
|
s = ""
|
||||||
|
longest_name = max([len(key) for key in result.keys()])
|
||||||
|
longest_match = max([len(m[0]) for m in result.values()])
|
||||||
|
|
||||||
|
for key in result.keys():
|
||||||
|
name = key + ": "
|
||||||
|
while len(name) <= 2+longest_name:
|
||||||
|
name += " "
|
||||||
|
|
||||||
|
match_list = result[key][0]
|
||||||
|
while len(match_list) <= longest_match:
|
||||||
|
match_list.append(" ")
|
||||||
|
matches = ""
|
||||||
|
for m in match_list:
|
||||||
|
matches += m
|
||||||
|
|
||||||
|
percentage = round(result[key][1]*100, 2)
|
||||||
|
if not small:
|
||||||
|
s += name + "Matches: " + matches + " MatchPercent: " + str(percentage) + "\n"
|
||||||
|
else:
|
||||||
|
s += name + str(percentage) + "\n"
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
kanji = Alphabet("Kanji (small)", "kanji.csv")
|
||||||
|
kanji2 = Alphabet("Kanji (big)", "kanji2.csv")
|
||||||
|
hiragana = Alphabet("Hiragana", "hiragana.csv")
|
||||||
|
katakana = Alphabet("Katakana", "katakana.csv")
|
||||||
|
print(kanji.get_name(), len(kanji.get_data()))
|
||||||
|
print(kanji2.get_name(), len(kanji2.get_data()))
|
||||||
|
print(hiragana.get_name(), len(hiragana.get_data()))
|
||||||
|
print(katakana.get_name(), len(katakana.get_data()))
|
||||||
|
print()
|
||||||
|
|
||||||
|
d = Detector()
|
||||||
|
d.add_alphabet(kanji)
|
||||||
|
d.add_alphabet(kanji2)
|
||||||
|
d.add_alphabet(hiragana)
|
||||||
|
d.add_alphabet(katakana)
|
||||||
|
|
||||||
|
|
||||||
|
text = ""
|
||||||
|
for line in open("./sample-text.txt", 'r'):
|
||||||
|
text += line
|
||||||
|
|
||||||
|
print("Matches: \n{0}".format(d.pretty_result(d.match_text(text), small=True)))
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
Char,Unicode (hex),Decimal,LearningOrder
|
||||||
|
ぁ,U+3041,12353,
|
||||||
|
あ,U+3042,12354,
|
||||||
|
ぃ,U+3043,12355,
|
||||||
|
い,U+3044,12356,
|
||||||
|
ぅ,U+3045,12357,
|
||||||
|
う,U+3046,12358,
|
||||||
|
ぇ,U+3047,12359,
|
||||||
|
え,U+3048,12360,
|
||||||
|
ぉ,U+3049,12361,
|
||||||
|
お,U+304a,12362,
|
||||||
|
か,U+304b,12363,
|
||||||
|
が,U+304c,12364,
|
||||||
|
き,U+304d,12365,
|
||||||
|
ぎ,U+304e,12366,
|
||||||
|
く,U+304f,12367,
|
||||||
|
ぐ,U+3050,12368,
|
||||||
|
け,U+3051,12369,
|
||||||
|
げ,U+3052,12370,
|
||||||
|
こ,U+3053,12371,
|
||||||
|
ご,U+3054,12372,
|
||||||
|
さ,U+3055,12373,
|
||||||
|
ざ,U+3056,12374,
|
||||||
|
し,U+3057,12375,
|
||||||
|
す,U+3059,12377,
|
||||||
|
ず,U+305a,12378,
|
||||||
|
せ,U+305b,12379,
|
||||||
|
ぜ,U+305c,12380,
|
||||||
|
そ,U+305d,12381,
|
||||||
|
ぞ,U+305e,12382,
|
||||||
|
た,U+305f,12383,
|
||||||
|
だ,U+3060,12384,
|
||||||
|
ち,U+3061,12385,
|
||||||
|
ぢ,U+3062,12386,
|
||||||
|
っ,U+3063,12387,
|
||||||
|
つ,U+3064,12388,
|
||||||
|
づ,U+3065,12389,
|
||||||
|
て,U+3066,12390,
|
||||||
|
で,U+3067,12391,
|
||||||
|
ど,U+3069,12393,
|
||||||
|
な,U+306a,12394,
|
||||||
|
に,U+306b,12395,
|
||||||
|
ぬ,U+306c,12396,
|
||||||
|
ね,U+306d,12397,
|
||||||
|
の,U+306e,12398,
|
||||||
|
は,U+306f,12399,
|
||||||
|
ば,U+3070,12400,
|
||||||
|
ぱ,U+3071,12401,
|
||||||
|
ひ,U+3072,12402,
|
||||||
|
び,U+3073,12403,
|
||||||
|
ぴ,U+3074,12404,
|
||||||
|
ふ,U+3075,12405,
|
||||||
|
ぶ,U+3076,12406,
|
||||||
|
ぷ,U+3077,12407,
|
||||||
|
べ,U+3079,12409,
|
||||||
|
ぺ,U+307a,12410,
|
||||||
|
ほ,U+307b,12411,
|
||||||
|
ぼ,U+307c,12412,
|
||||||
|
ぽ,U+307d,12413,
|
||||||
|
ま,U+307e,12414,
|
||||||
|
み,U+307f,12415,
|
||||||
|
む,U+3080,12416,
|
||||||
|
め,U+3081,12417,
|
||||||
|
も,U+3082,12418,
|
||||||
|
ゃ,U+3083,12419,
|
||||||
|
や,U+3084,12420,
|
||||||
|
ゅ,U+3085,12421,
|
||||||
|
ゆ,U+3086,12422,
|
||||||
|
ょ,U+3087,12423,
|
||||||
|
ら,U+3089,12425,
|
||||||
|
り,U+308a,12426,
|
||||||
|
る,U+308b,12427,
|
||||||
|
れ,U+308c,12428,
|
||||||
|
ろ,U+308d,12429,
|
||||||
|
ゎ,U+308e,12430,
|
||||||
|
わ,U+308f,12431,
|
||||||
|
ゐ,U+3090,12432,
|
||||||
|
ゑ,U+3091,12433,
|
||||||
|
を,U+3092,12434,
|
||||||
|
ん,U+3093,12435,
|
||||||
|
ゔ,U+3094,12436,
|
||||||
|
ゕ,U+3095,12437,
|
||||||
|
ゖ,U+3096,12438,
|
||||||
|
゙,U+3099,12441,
|
||||||
|
゚,U+309a,12442,
|
||||||
|
゛,U+309b,12443,
|
||||||
|
゜,U+309c,12444,
|
||||||
|
ゝ,U+309d,12445,
|
||||||
|
ゞ,U+309e,12446,
|
||||||
|
ゟ,U+309f,12447,
|
||||||
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,5 @@
|
|||||||
|
会市屋詩
|
||||||
|
|
||||||
|
ヷムペヺ
|
||||||
|
|
||||||
|
ぅぴ
|
||||||
Loading…
Reference in New Issue