From db8a4ba13192c4be88ef18dfb8af22a9ca122991 Mon Sep 17 00:00:00 2001 From: RedoC-github Date: Sun, 29 Nov 2020 23:13:43 +0900 Subject: [PATCH 1/2] change to c++ because of module error. --- PLAN.md | 2 +- src/stream/CI_TEST.py | 5 ++ src/stream/keywords.py | 21 ------ src/stream/parse.py | 57 --------------- src/stream/parse_test.py | 5 -- src/stream/proto/basic.streamcode | 5 -- src/stream/proto/basic_parsed.streamparse | 31 -------- src/stream/stream.py | 88 ----------------------- src/stream/stream_test.py | 10 --- src/stream/string_obfuscation.py | 22 ------ 10 files changed, 6 insertions(+), 240 deletions(-) delete mode 100644 src/stream/keywords.py delete mode 100644 src/stream/parse.py delete mode 100644 src/stream/parse_test.py delete mode 100644 src/stream/proto/basic.streamcode delete mode 100644 src/stream/proto/basic_parsed.streamparse delete mode 100644 src/stream/stream.py delete mode 100644 src/stream/stream_test.py delete mode 100644 src/stream/string_obfuscation.py diff --git a/PLAN.md b/PLAN.md index 5a6a823..a83cef3 100644 --- a/PLAN.md +++ b/PLAN.md @@ -4,7 +4,7 @@ |✅|75% :octocat:||0% :octocat:||| |:-:|:-:|:-:|:-:|:-:|:-:| -|**Start**|L1 parser|L2 parser *(only for PERFORM. See [stream.py](./src/stream/stream.py))*|syntax checker|runtime|**Release**| +|**Start**|L1 parser|L2 parser *(only for PERFORM. See [stream.py](./src/stream/parser/stream.py))*|syntax checker|runtime|**Release**| ✅ : Finished :octocat: : Working diff --git a/src/stream/CI_TEST.py b/src/stream/CI_TEST.py index a8a4ad8..66a0241 100644 --- a/src/stream/CI_TEST.py +++ b/src/stream/CI_TEST.py @@ -7,4 +7,9 @@ Mention @Diggie-Bro/froglang for any help! """ +# L1 parser +from .parser import parse + +parse.Generator() + diff --git a/src/stream/keywords.py b/src/stream/keywords.py deleted file mode 100644 index 1792896..0000000 --- a/src/stream/keywords.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -StreamCode Prototype, under Froglang - -keywords.py -Stream Grammar Keywords file. - -Mention @Diggie-Bro/froglang for any help! -""" - -stream_keyword = { - "PROCPIPE": ">>", # process pipeline - "RETURNTOKEN": "::", # return token - "PARAMTOKEN": "->", # param token - "TAGTOKEN": "|", # tag token - "VARSTREAMTOK": ":=" # varStream token -} - -dummy_keyword = { - "OBFUSSTRING": "&#$@@@", - "MULTILENTER": "&#$###" -} diff --git a/src/stream/parse.py b/src/stream/parse.py deleted file mode 100644 index 20b8b6f..0000000 --- a/src/stream/parse.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -StreamCode Prototype, under Froglang - -parse.py -Stream parsed file generator. - -Mention @Diggie-Bro/froglang for any help! -""" -import stream - -class Generator: - def __init__(self, codepath, targetpath): - self.tgpath = targetpath - self.cdpath = codepath - - def convertStructToParse(self, structs, meta): - string = """ -startline -IS_VARIABLE_STREAM {isvs} -VARIABLE_STREAM_NAME {vsname} - """.format( - isvs=meta['variableStream'], - vsname=meta['vsName'] - ) - - for struct in structs: - string += """ -startstream {order}|{tag} -PARAM {param} -RETURN {return_value} -PERFORM {perform} -endstream - """.format( - order=struct['order'], - param=struct['param'], - return_value=struct['return_value'], - perform=struct['operation'], - tag=struct['tag'] - ) - - string += "endline" - - return string - - def generateParsedFile(self): - parsed_string = "" - - with open(self.cdpath, 'r') as f: - for linecode in '\n'.join([e.strip() for e in f.readlines()]).split(";")[:-1]: - strm = stream.Stream(linecode) - strm.makeStructure() - - # parsing - parsed_string += self.convertStructToParse(strm.stream, strm.metadata) - - with open(self.tgpath, 'w') as f: - f.write(parsed_string) diff --git a/src/stream/parse_test.py b/src/stream/parse_test.py deleted file mode 100644 index 5a31809..0000000 --- a/src/stream/parse_test.py +++ /dev/null @@ -1,5 +0,0 @@ -import parse - -if __name__ == '__main__': - parser = parse.Generator(codepath="proto/basic.streamcode", targetpath="proto/basic_parsed.streamparse") - parser.generateParsedFile() diff --git a/src/stream/proto/basic.streamcode b/src/stream/proto/basic.streamcode deleted file mode 100644 index f078bf6..0000000 --- a/src/stream/proto/basic.streamcode +++ /dev/null @@ -1,5 +0,0 @@ -hello_name := name->filtered_name::{ - filtered_name = "Hello, " + name - filtered_name += "Nice to meet you!" -}>> filtered_name->run print(filtered_name); -main := name::name = run input("What's your name?") >> hello_name; diff --git a/src/stream/proto/basic_parsed.streamparse b/src/stream/proto/basic_parsed.streamparse deleted file mode 100644 index 5dc09d1..0000000 --- a/src/stream/proto/basic_parsed.streamparse +++ /dev/null @@ -1,31 +0,0 @@ -startline -IS_VARIABLE_STREAM True -VARIABLE_STREAM_NAME hello_name - -startstream 0|None -PARAM name -RETURN filtered_name -PERFORM {&#$###filtered_name = "72&#$@@@101&#$@@@108&#$@@@108&#$@@@111&35&#$@@@36&#$@@@64&#$@@@64&#$@@@64&#$@@@52&#$@@@52&#$@@@38&#$@@@35&#$@@@36&#$@@@64&#$@@@64&#$@@@64&#$@@@51&#$@@@50&#$@@@34&#$@@@32+ name&#$###filtered_name += "Nice to meet you!"&#$###} -endstream - -startstream 1|None -PARAM filtered_name -RETURN None -PERFORM run print(filtered_name) -endstream - endlinestartline -IS_VARIABLE_STREAM True -VARIABLE_STREAM_NAME main - -startstream 0|None -PARAM None -RETURN name -PERFORM name = run input("87&#$@@@104&#$@@@97&#$@@@116&#$@@@39&#$@@@115&#$@@@32&#$@@@121&#$@@@111&#$@@@117&#$@@@114&#$@@@32&#$@@@110&#$@@@97&#$@@@109&#$@@@101&#$@@@63") -endstream - -startstream 1|None -PARAM None -RETURN None -PERFORM hello_name -endstream - endline \ No newline at end of file diff --git a/src/stream/stream.py b/src/stream/stream.py deleted file mode 100644 index 205453e..0000000 --- a/src/stream/stream.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -StreamCode Prototype, under Froglang - -stream.py -Stream Structure Object File - -Mention @Diggie-Bro/froglang for any help! -""" -import re -import keywords -import string_obfuscation as strobfus - -""" -Stream Class -Stream Structure Object ----- -Generate, and return Stream structure. - -:param code: str -""" -class Stream: - def __init__(self, code: str): - self.code = code - self.stream = [] - self.metadata = {} - - def makeStructure(self): - self.metadata = { - "variableStream": True if ":=" in self.code else False, - "vsName": self.code.split(":=")[0].strip() if ":=" in self.code else None - } - codes = self.code.split(">>") - - # Stream structure - # { variableStream, vsName, operation, param, return_value, order, tag } - - for index, code in enumerate(codes): - code.strip() - - # string obfuscation not to confuse during parsing. - # string only support ", not ' - # multiline support, too - stringRangeIndex: int = None - braceRangeIndex: int = None - - for i, char in enumerate(list(code)): - if char == '{': - if braceRangeIndex == None: - braceRangeIndex = i - elif char == '}': - # suppose grammar is right. - if braceRangeIndex != None: - multilPart = code[braceRangeIndex + 1: i] - multilPart_edited = multilPart.replace("\n", keywords.dummy_keyword["MULTILENTER"]).strip() - code = code.replace(multilPart, multilPart_edited) - braceRangeIndex = None - - for i, char in enumerate(list(code)): - if char == '"': - if stringRangeIndex != None: - stringPart = code[stringRangeIndex + 1: i] - code = code.replace(stringPart, strobfus.encode(stringPart)) - stringRangeIndex = None - else: - stringRangeIndex = i - - self.stream.append({ - "operation": code.split("::")[1].strip() if "::" in code else (code.split("->")[1].strip() if "->" in - code else code.strip()), - "param": ( - code.split("|")[1].split("->")[0].strip() if "|" in code else ( - code.split(":=")[1].split("->")[0].strip() if ":=" in code else code.split("->")[0].strip() - ) - ) if "->" in code else None, - "return_value": ( - code.split("::")[0].split("->")[1].strip() if "->" in code else ( - code.split("::")[0].split("|")[1].strip() if "|" in code else ( - code.split("::")[0].split(":=")[1].strip() if ":=" in code else - code.split("::")[0].strip() - ) - ) - ) if "::" in code else None, - "order": index, - "tag": ( - code.split(":=")[1].split("|")[0].strip() if ":=" in code else code.split("|")[0].strip() - ) if "|" in code else None - }) - diff --git a/src/stream/stream_test.py b/src/stream/stream_test.py deleted file mode 100644 index d89f920..0000000 --- a/src/stream/stream_test.py +++ /dev/null @@ -1,10 +0,0 @@ -import stream - -if __name__ == '__main__': - strm = stream.Stream( - """IO := I|name::name = run input("What's your name?") >> O|name->run print("Hello, " + name)""" - ) - strm.makeStructure() - print(strm.metadata) - print(strm.stream) - diff --git a/src/stream/string_obfuscation.py b/src/stream/string_obfuscation.py deleted file mode 100644 index 85c67ff..0000000 --- a/src/stream/string_obfuscation.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -StreamCode Prototype, under Froglang - -string_obfuscation.py -Stream String Obfuscation file. - -Notice! This is not for security. - -Mention @Diggie-Bro/froglang for any help! -""" - -import keywords - -def encode(s): - return keywords.dummy_keyword["OBFUSSTRING"].join([str(ord(c)) for c in list(s)]) - -def decode(s: str): - for splited in s.split(keywords.dummy_keyword["OBFUSSTRING"]): - s = s.replace(splited, chr(int(splited))) - - return s.replace(keywords.dummy_keyword["OBFUSSTRING"], '') - From f308cf0d7df637e56f820ba403effb0b745c4310 Mon Sep 17 00:00:00 2001 From: RedoC-github Date: Sun, 29 Nov 2020 23:14:52 +0900 Subject: [PATCH 2/2] misc --- src/stream/__init__.py | 0 src/stream/parser/__init__.py | 1 + src/stream/parser/keywords.py | 21 +++++ src/stream/parser/parse.py | 57 ++++++++++++ src/stream/parser/parse_test.py | 5 ++ src/stream/parser/proto/basic.streamcode | 5 ++ .../parser/proto/basic_parsed.streamparse | 31 +++++++ src/stream/parser/stream.py | 88 +++++++++++++++++++ src/stream/parser/stream_test.py | 10 +++ src/stream/parser/string_obfuscation.py | 22 +++++ 10 files changed, 240 insertions(+) create mode 100644 src/stream/__init__.py create mode 100644 src/stream/parser/__init__.py create mode 100644 src/stream/parser/keywords.py create mode 100644 src/stream/parser/parse.py create mode 100644 src/stream/parser/parse_test.py create mode 100644 src/stream/parser/proto/basic.streamcode create mode 100644 src/stream/parser/proto/basic_parsed.streamparse create mode 100644 src/stream/parser/stream.py create mode 100644 src/stream/parser/stream_test.py create mode 100644 src/stream/parser/string_obfuscation.py diff --git a/src/stream/__init__.py b/src/stream/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/stream/parser/__init__.py b/src/stream/parser/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/stream/parser/__init__.py @@ -0,0 +1 @@ + diff --git a/src/stream/parser/keywords.py b/src/stream/parser/keywords.py new file mode 100644 index 0000000..e74c935 --- /dev/null +++ b/src/stream/parser/keywords.py @@ -0,0 +1,21 @@ +""" +StreamCode Prototype, under Froglang + +keywords.py +Stream Grammar Keywords File for parsing + +Mention @Diggie-Bro/froglang for any help! +""" + +stream_keyword = { + "PROCPIPE": ">>", # process pipeline + "RETURNTOKEN": "::", # return token + "PARAMTOKEN": "->", # param token + "TAGTOKEN": "|", # tag token + "VARSTREAMTOK": ":=" # varStream token +} + +dummy_keyword = { + "OBFUSSTRING": "&#$@@@", + "MULTILENTER": "&#$###" +} diff --git a/src/stream/parser/parse.py b/src/stream/parser/parse.py new file mode 100644 index 0000000..9f1486c --- /dev/null +++ b/src/stream/parser/parse.py @@ -0,0 +1,57 @@ +""" +StreamCode Prototype, under Froglang + +parse.py +Stream parsed file generator. + +Mention @Diggie-Bro/froglang for any help! +""" +from . import stream + +class Generator: + def __init__(self, codepath, targetpath): + self.tgpath = targetpath + self.cdpath = codepath + + def convertStructToParse(self, structs, meta): + string = """ +startline +IS_VARIABLE_STREAM {isvs} +VARIABLE_STREAM_NAME {vsname} + """.format( + isvs=meta['variableStream'], + vsname=meta['vsName'] + ) + + for struct in structs: + string += """ +startstream {order}|{tag} +PARAM {param} +RETURN {return_value} +PERFORM {perform} +endstream + """.format( + order=struct['order'], + param=struct['param'], + return_value=struct['return_value'], + perform=struct['operation'], + tag=struct['tag'] + ) + + string += "endline" + + return string + + def generateParsedFile(self): + parsed_string = "" + + with open(self.cdpath, 'r') as f: + for linecode in '\n'.join([e.strip() for e in f.readlines()]).split(";")[:-1]: + strm = stream.Stream(linecode) + strm.makeStructure() + + # parsing + parsed_string += self.convertStructToParse(strm.stream, strm.metadata) + + with open(self.tgpath, 'w') as f: + f.write(parsed_string) diff --git a/src/stream/parser/parse_test.py b/src/stream/parser/parse_test.py new file mode 100644 index 0000000..5381621 --- /dev/null +++ b/src/stream/parser/parse_test.py @@ -0,0 +1,5 @@ +from . import parse + +if __name__ == '__main__': + parser = parse.Generator(codepath="proto/basic.streamcode", targetpath="proto/basic_parsed.streamparse") + parser.generateParsedFile() diff --git a/src/stream/parser/proto/basic.streamcode b/src/stream/parser/proto/basic.streamcode new file mode 100644 index 0000000..f078bf6 --- /dev/null +++ b/src/stream/parser/proto/basic.streamcode @@ -0,0 +1,5 @@ +hello_name := name->filtered_name::{ + filtered_name = "Hello, " + name + filtered_name += "Nice to meet you!" +}>> filtered_name->run print(filtered_name); +main := name::name = run input("What's your name?") >> hello_name; diff --git a/src/stream/parser/proto/basic_parsed.streamparse b/src/stream/parser/proto/basic_parsed.streamparse new file mode 100644 index 0000000..5dc09d1 --- /dev/null +++ b/src/stream/parser/proto/basic_parsed.streamparse @@ -0,0 +1,31 @@ +startline +IS_VARIABLE_STREAM True +VARIABLE_STREAM_NAME hello_name + +startstream 0|None +PARAM name +RETURN filtered_name +PERFORM {&#$###filtered_name = "72&#$@@@101&#$@@@108&#$@@@108&#$@@@111&35&#$@@@36&#$@@@64&#$@@@64&#$@@@64&#$@@@52&#$@@@52&#$@@@38&#$@@@35&#$@@@36&#$@@@64&#$@@@64&#$@@@64&#$@@@51&#$@@@50&#$@@@34&#$@@@32+ name&#$###filtered_name += "Nice to meet you!"&#$###} +endstream + +startstream 1|None +PARAM filtered_name +RETURN None +PERFORM run print(filtered_name) +endstream + endlinestartline +IS_VARIABLE_STREAM True +VARIABLE_STREAM_NAME main + +startstream 0|None +PARAM None +RETURN name +PERFORM name = run input("87&#$@@@104&#$@@@97&#$@@@116&#$@@@39&#$@@@115&#$@@@32&#$@@@121&#$@@@111&#$@@@117&#$@@@114&#$@@@32&#$@@@110&#$@@@97&#$@@@109&#$@@@101&#$@@@63") +endstream + +startstream 1|None +PARAM None +RETURN None +PERFORM hello_name +endstream + endline \ No newline at end of file diff --git a/src/stream/parser/stream.py b/src/stream/parser/stream.py new file mode 100644 index 0000000..f2872f9 --- /dev/null +++ b/src/stream/parser/stream.py @@ -0,0 +1,88 @@ +""" +StreamCode Prototype, under Froglang + +stream.py +Stream Structure Object File for Parsing + +Mention @Diggie-Bro/froglang for any help! +""" +from . import keywords +from . import string_obfuscation as strobfus + +""" +Stream Class +Stream Structure Object +---- +Generate, and return Stream structure. + +:param code: str +""" + + +class Stream: + def __init__(self, code: str): + self.code = code + self.stream = [] + self.metadata = {} + + def makeStructure(self): + self.metadata = { + "variableStream": True if ":=" in self.code else False, + "vsName": self.code.split(":=")[0].strip() if ":=" in self.code else None + } + codes = self.code.split(">>") + + # Stream structure + # { variableStream, vsName, operation, param, return_value, order, tag } + + for index, code in enumerate(codes): + code.strip() + + # string obfuscation not to confuse during parsing. + # string only support ", not ' + # multiline support, too + stringRangeIndex: int = None + braceRangeIndex: int = None + + for i, char in enumerate(list(code)): + if char == '{': + if braceRangeIndex is None: + braceRangeIndex = i + elif char == '}': + # suppose grammar is right. + if braceRangeIndex is not None: + multilPart = code[braceRangeIndex + 1: i] + multilPart_edited = multilPart.replace("\n", keywords.dummy_keyword["MULTILENTER"]).strip() + code = code.replace(multilPart, multilPart_edited) + braceRangeIndex = None + for i, char in enumerate(list(code)): + if char == '"': + if stringRangeIndex is not None: + stringPart = code[stringRangeIndex + 1: i] + code = code.replace(stringPart, strobfus.encode(stringPart)) + stringRangeIndex = None + else: + stringRangeIndex = i + + self.stream.append({ + "operation": code.split("::")[1].strip() if "::" in code else (code.split("->")[1].strip() if "->" in + code else code.strip()), + "param": ( + code.split("|")[1].split("->")[0].strip() if "|" in code else ( + code.split(":=")[1].split("->")[0].strip() if ":=" in code else code.split("->")[0].strip() + ) + ) if "->" in code else None, + "return_value": ( + code.split("::")[0].split("->")[1].strip() if "->" in code else ( + code.split("::")[0].split("|")[1].strip() if "|" in code else ( + code.split("::")[0].split(":=")[1].strip() if ":=" in code else + code.split("::")[0].strip() + ) + ) + ) if "::" in code else None, + "order": index, + "tag": ( + code.split(":=")[1].split("|")[0].strip() if ":=" in code else code.split("|")[0].strip() + ) if "|" in code else None + }) + diff --git a/src/stream/parser/stream_test.py b/src/stream/parser/stream_test.py new file mode 100644 index 0000000..280b365 --- /dev/null +++ b/src/stream/parser/stream_test.py @@ -0,0 +1,10 @@ +from . import stream + +if __name__ == '__main__': + strm = stream.Stream( + """IO := I|name::name = run input("What's your name?") >> O|name->run print("Hello, " + name)""" + ) + strm.makeStructure() + print(strm.metadata) + print(strm.stream) + diff --git a/src/stream/parser/string_obfuscation.py b/src/stream/parser/string_obfuscation.py new file mode 100644 index 0000000..e4ba42f --- /dev/null +++ b/src/stream/parser/string_obfuscation.py @@ -0,0 +1,22 @@ +""" +StreamCode Prototype, under Froglang + +string_obfuscation.py +Stream String Obfuscation file. + +Notice! This is not for security. + +Mention @Diggie-Bro/froglang for any help! +""" + +from . import keywords + +def encode(s): + return keywords.dummy_keyword["OBFUSSTRING"].join([str(ord(c)) for c in list(s)]) + +def decode(s: str): + for splited in s.split(keywords.dummy_keyword["OBFUSSTRING"]): + s = s.replace(splited, chr(int(splited))) + + return s.replace(keywords.dummy_keyword["OBFUSSTRING"], '') +